diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index b34a2a4f3a7d6..511679ef71a79 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -168,10 +168,10 @@ export class Plugin implements ISecuritySolutionPlugin { initUsageCollectors({ core, - kibanaIndex: core.savedObjects.getKibanaIndex(), signalsIndex: DEFAULT_ALERTS_INDEX, ml: plugins.ml, usageCollection: plugins.usageCollection, + logger, }); this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(APP_ID); diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts index 4530dac725c7b..dc98b68f9f186 100644 --- a/x-pack/plugins/security_solution/server/usage/collector.ts +++ b/x-pack/plugins/security_solution/server/usage/collector.ts @@ -5,38 +5,26 @@ * 2.0. */ -import { CoreSetup, SavedObjectsClientContract } from '../../../../../src/core/server'; -import { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; -import { CollectorDependencies } from './types'; -import { fetchDetectionsMetrics } from './detections'; -import { SAVED_OBJECT_TYPES } from '../../../cases/common/constants'; -// eslint-disable-next-line no-restricted-imports -import { legacyRuleActionsSavedObjectType } from '../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; +import type { CollectorFetchContext } from '../../../../../src/plugins/usage_collection/server'; +import type { CollectorDependencies } from './types'; +import { getDetectionsMetrics } from './detections/get_metrics'; +import { getInternalSavedObjectsClient } from './get_internal_saved_objects_client'; export type RegisterCollector = (deps: CollectorDependencies) => void; + export interface UsageData { detectionMetrics: {}; } -export async function getInternalSavedObjectsClient(core: CoreSetup) { - return core.getStartServices().then(async ([coreStart]) => { - // note: we include the "cases" and "alert" hidden types here otherwise we would not be able to query them. If at some point cases and alert is not considered a hidden type this can be removed - return coreStart.savedObjects.createInternalRepository([ - 'alert', - legacyRuleActionsSavedObjectType, - ...SAVED_OBJECT_TYPES, - ]); - }); -} - export const registerCollector: RegisterCollector = ({ core, - kibanaIndex, signalsIndex, ml, usageCollection, + logger, }) => { if (!usageCollection) { + logger.debug('Usage collection is undefined, therefore returning early without registering it'); return; } @@ -525,12 +513,16 @@ export const registerCollector: RegisterCollector = ({ }, isReady: () => true, fetch: async ({ esClient }: CollectorFetchContext): Promise => { - const internalSavedObjectsClient = await getInternalSavedObjectsClient(core); - const soClient = internalSavedObjectsClient as unknown as SavedObjectsClientContract; - + const savedObjectsClient = await getInternalSavedObjectsClient(core); + const detectionMetrics = await getDetectionsMetrics({ + signalsIndex, + esClient, + savedObjectsClient, + logger, + mlClient: ml, + }); return { - detectionMetrics: - (await fetchDetectionsMetrics(kibanaIndex, signalsIndex, esClient, soClient, ml)) || {}, + detectionMetrics: detectionMetrics || {}, }; }, }); diff --git a/x-pack/plugins/security_solution/server/usage/constants.ts b/x-pack/plugins/security_solution/server/usage/constants.ts new file mode 100644 index 0000000000000..d3d526768fcd5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/constants.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * We limit the max results window to prevent in-memory from blowing up when we do correlation. + * This is limiting us to 10,000 cases and 10,000 elastic detection rules to do telemetry and correlation + * and the choice was based on the initial "index.max_result_window" before this turned into a PIT (Point In Time) + * implementation. + * + * This number could be changed, and the implementation details of how we correlate could change as well (maybe) + * to avoid pulling 10,000 worth of cases and elastic rules into memory. + * + * However, for now, we are keeping this maximum as the original and the in-memory implementation + */ +export const MAX_RESULTS_WINDOW = 10_000; + +/** + * We choose our max per page based on 1k as that + * appears to be what others are choosing here in the other sections of telemetry: + * https://github.com/elastic/kibana/pull/99031 + */ +export const MAX_PER_PAGE = 1_000; diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts deleted file mode 100644 index 1aadcfdc5478a..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.ts +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { KibanaRequest, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { MlDatafeedStats, MlJob, MlPluginSetup } from '../../../../ml/server'; -import { isJobStarted } from '../../../common/machine_learning/helpers'; -import { isSecurityJob } from '../../../common/machine_learning/is_security_job'; -import { DetectionsMetric, MlJobMetric, MlJobsUsage, MlJobUsage } from './types'; - -/** - * Default ml job usage count - */ -export const initialMlJobsUsage: MlJobsUsage = { - custom: { - enabled: 0, - disabled: 0, - }, - elastic: { - enabled: 0, - disabled: 0, - }, -}; - -export const updateMlJobsUsage = (jobMetric: DetectionsMetric, usage: MlJobsUsage): MlJobsUsage => { - const { isEnabled, isElastic } = jobMetric; - if (isEnabled && isElastic) { - return { - ...usage, - elastic: { - ...usage.elastic, - enabled: usage.elastic.enabled + 1, - }, - }; - } else if (!isEnabled && isElastic) { - return { - ...usage, - elastic: { - ...usage.elastic, - disabled: usage.elastic.disabled + 1, - }, - }; - } else if (isEnabled && !isElastic) { - return { - ...usage, - custom: { - ...usage.custom, - enabled: usage.custom.enabled + 1, - }, - }; - } else if (!isEnabled && !isElastic) { - return { - ...usage, - custom: { - ...usage.custom, - disabled: usage.custom.disabled + 1, - }, - }; - } else { - return usage; - } -}; - -export const getMlJobMetrics = async ( - ml: MlPluginSetup | undefined, - savedObjectClient: SavedObjectsClientContract -): Promise => { - let jobsUsage: MlJobsUsage = initialMlJobsUsage; - - if (ml) { - try { - const fakeRequest = { headers: {} } as KibanaRequest; - - const modules = await ml.modulesProvider(fakeRequest, savedObjectClient).listModules(); - const moduleJobs = modules.flatMap((module) => module.jobs); - const jobs = await ml.jobServiceProvider(fakeRequest, savedObjectClient).jobsSummary(); - - jobsUsage = jobs.filter(isSecurityJob).reduce((usage, job) => { - const isElastic = moduleJobs.some((moduleJob) => moduleJob.id === job.id); - const isEnabled = isJobStarted(job.jobState, job.datafeedState); - - return updateMlJobsUsage({ isElastic, isEnabled }, usage); - }, initialMlJobsUsage); - - const jobsType = 'security'; - const securityJobStats = await ml - .anomalyDetectorsProvider(fakeRequest, savedObjectClient) - .jobStats(jobsType); - - const jobDetails = await ml - .anomalyDetectorsProvider(fakeRequest, savedObjectClient) - .jobs(jobsType); - - const jobDetailsCache = new Map(); - jobDetails.jobs.forEach((detail) => jobDetailsCache.set(detail.job_id, detail)); - - const datafeedStats = await ml - .anomalyDetectorsProvider(fakeRequest, savedObjectClient) - .datafeedStats(); - - const datafeedStatsCache = new Map(); - datafeedStats.datafeeds.forEach((datafeedStat) => - datafeedStatsCache.set(`${datafeedStat.datafeed_id}`, datafeedStat) - ); - - const jobMetrics: MlJobMetric[] = securityJobStats.jobs.map((stat) => { - const jobId = stat.job_id; - const jobDetail = jobDetailsCache.get(stat.job_id); - const datafeed = datafeedStatsCache.get(`datafeed-${jobId}`); - - return { - job_id: jobId, - open_time: stat.open_time, - create_time: jobDetail?.create_time, - finished_time: jobDetail?.finished_time, - state: stat.state, - data_counts: { - bucket_count: stat.data_counts.bucket_count, - empty_bucket_count: stat.data_counts.empty_bucket_count, - input_bytes: stat.data_counts.input_bytes, - input_record_count: stat.data_counts.input_record_count, - last_data_time: stat.data_counts.last_data_time, - processed_record_count: stat.data_counts.processed_record_count, - }, - model_size_stats: { - bucket_allocation_failures_count: - stat.model_size_stats.bucket_allocation_failures_count, - memory_status: stat.model_size_stats.memory_status, - model_bytes: stat.model_size_stats.model_bytes, - model_bytes_exceeded: stat.model_size_stats.model_bytes_exceeded, - model_bytes_memory_limit: stat.model_size_stats.model_bytes_memory_limit, - peak_model_bytes: stat.model_size_stats.peak_model_bytes, - }, - timing_stats: { - average_bucket_processing_time_ms: stat.timing_stats.average_bucket_processing_time_ms, - bucket_count: stat.timing_stats.bucket_count, - exponential_average_bucket_processing_time_ms: - stat.timing_stats.exponential_average_bucket_processing_time_ms, - exponential_average_bucket_processing_time_per_hour_ms: - stat.timing_stats.exponential_average_bucket_processing_time_per_hour_ms, - maximum_bucket_processing_time_ms: stat.timing_stats.maximum_bucket_processing_time_ms, - minimum_bucket_processing_time_ms: stat.timing_stats.minimum_bucket_processing_time_ms, - total_bucket_processing_time_ms: stat.timing_stats.total_bucket_processing_time_ms, - }, - datafeed: { - datafeed_id: datafeed?.datafeed_id, - state: datafeed?.state, - timing_stats: { - bucket_count: datafeed?.timing_stats.bucket_count, - exponential_average_search_time_per_hour_ms: - datafeed?.timing_stats.exponential_average_search_time_per_hour_ms, - search_count: datafeed?.timing_stats.search_count, - total_search_time_ms: datafeed?.timing_stats.total_search_time_ms, - }, - }, - } as MlJobMetric; - }); - - return { - ml_job_usage: jobsUsage, - ml_job_metrics: jobMetrics, - }; - } catch (e) { - // ignore failure, usage will be zeroed - } - } - - return { - ml_job_usage: initialMlJobsUsage, - ml_job_metrics: [], - }; -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts deleted file mode 100644 index 39c108931e2d7..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.ts +++ /dev/null @@ -1,503 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - SIGNALS_ID, - EQL_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, -} from '@kbn/securitysolution-rules'; -import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; -import { CASE_COMMENT_SAVED_OBJECT } from '../../../../cases/common/constants'; - -import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { isElasticRule } from './index'; -import type { - AlertsAggregationResponse, - CasesSavedObject, - DetectionRulesTypeUsage, - DetectionRuleMetric, - DetectionRuleAdoption, - RuleSearchParams, - RuleSearchResult, - DetectionMetrics, -} from './types'; -// eslint-disable-next-line no-restricted-imports -import { legacyRuleActionsSavedObjectType } from '../../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; -// eslint-disable-next-line no-restricted-imports -import { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../lib/detection_engine/rule_actions/legacy_types'; - -/** - * Initial detection metrics initialized. - */ -export const getInitialDetectionMetrics = (): DetectionMetrics => ({ - ml_jobs: { - ml_job_usage: { - custom: { - enabled: 0, - disabled: 0, - }, - elastic: { - enabled: 0, - disabled: 0, - }, - }, - ml_job_metrics: [], - }, - detection_rules: { - detection_rule_detail: [], - detection_rule_usage: initialDetectionRulesUsage, - }, -}); - -/** - * Default detection rule usage count, split by type + elastic/custom - */ -export const initialDetectionRulesUsage: DetectionRulesTypeUsage = { - query: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - threshold: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - eql: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - machine_learning: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - threat_match: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - elastic_total: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, - custom_total: { - enabled: 0, - disabled: 0, - alerts: 0, - cases: 0, - legacy_notifications_enabled: 0, - legacy_notifications_disabled: 0, - notifications_enabled: 0, - notifications_disabled: 0, - }, -}; - -/* eslint-disable complexity */ -export const updateDetectionRuleUsage = ( - detectionRuleMetric: DetectionRuleMetric, - usage: DetectionRulesTypeUsage -): DetectionRulesTypeUsage => { - let updatedUsage = usage; - - const legacyNotificationEnabled = - detectionRuleMetric.has_legacy_notification && detectionRuleMetric.enabled; - - const legacyNotificationDisabled = - detectionRuleMetric.has_legacy_notification && !detectionRuleMetric.enabled; - - const notificationEnabled = detectionRuleMetric.has_notification && detectionRuleMetric.enabled; - - const notificationDisabled = detectionRuleMetric.has_notification && !detectionRuleMetric.enabled; - - if (detectionRuleMetric.rule_type === 'query') { - updatedUsage = { - ...usage, - query: { - ...usage.query, - enabled: detectionRuleMetric.enabled ? usage.query.enabled + 1 : usage.query.enabled, - disabled: !detectionRuleMetric.enabled ? usage.query.disabled + 1 : usage.query.disabled, - alerts: usage.query.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.query.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.query.legacy_notifications_enabled + 1 - : usage.query.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.query.legacy_notifications_disabled + 1 - : usage.query.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.query.notifications_enabled + 1 - : usage.query.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.query.notifications_disabled + 1 - : usage.query.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'threshold') { - updatedUsage = { - ...usage, - threshold: { - ...usage.threshold, - enabled: detectionRuleMetric.enabled - ? usage.threshold.enabled + 1 - : usage.threshold.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.threshold.disabled + 1 - : usage.threshold.disabled, - alerts: usage.threshold.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.threshold.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.threshold.legacy_notifications_enabled + 1 - : usage.threshold.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.threshold.legacy_notifications_disabled + 1 - : usage.threshold.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.threshold.notifications_enabled + 1 - : usage.threshold.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.threshold.notifications_disabled + 1 - : usage.threshold.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'eql') { - updatedUsage = { - ...usage, - eql: { - ...usage.eql, - enabled: detectionRuleMetric.enabled ? usage.eql.enabled + 1 : usage.eql.enabled, - disabled: !detectionRuleMetric.enabled ? usage.eql.disabled + 1 : usage.eql.disabled, - alerts: usage.eql.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.eql.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.eql.legacy_notifications_enabled + 1 - : usage.eql.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.eql.legacy_notifications_disabled + 1 - : usage.eql.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.eql.notifications_enabled + 1 - : usage.eql.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.eql.notifications_disabled + 1 - : usage.eql.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'machine_learning') { - updatedUsage = { - ...usage, - machine_learning: { - ...usage.machine_learning, - enabled: detectionRuleMetric.enabled - ? usage.machine_learning.enabled + 1 - : usage.machine_learning.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.machine_learning.disabled + 1 - : usage.machine_learning.disabled, - alerts: usage.machine_learning.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.machine_learning.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.machine_learning.legacy_notifications_enabled + 1 - : usage.machine_learning.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.machine_learning.legacy_notifications_disabled + 1 - : usage.machine_learning.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.machine_learning.notifications_enabled + 1 - : usage.machine_learning.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.machine_learning.notifications_disabled + 1 - : usage.machine_learning.notifications_disabled, - }, - }; - } else if (detectionRuleMetric.rule_type === 'threat_match') { - updatedUsage = { - ...usage, - threat_match: { - ...usage.threat_match, - enabled: detectionRuleMetric.enabled - ? usage.threat_match.enabled + 1 - : usage.threat_match.enabled, - disabled: !detectionRuleMetric.enabled - ? usage.threat_match.disabled + 1 - : usage.threat_match.disabled, - alerts: usage.threat_match.alerts + detectionRuleMetric.alert_count_daily, - cases: usage.threat_match.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? usage.threat_match.legacy_notifications_enabled + 1 - : usage.threat_match.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? usage.threat_match.legacy_notifications_disabled + 1 - : usage.threat_match.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? usage.threat_match.notifications_enabled + 1 - : usage.threat_match.notifications_enabled, - notifications_disabled: notificationDisabled - ? usage.threat_match.notifications_disabled + 1 - : usage.threat_match.notifications_disabled, - }, - }; - } - - if (detectionRuleMetric.elastic_rule) { - updatedUsage = { - ...updatedUsage, - elastic_total: { - ...updatedUsage.elastic_total, - enabled: detectionRuleMetric.enabled - ? updatedUsage.elastic_total.enabled + 1 - : updatedUsage.elastic_total.enabled, - disabled: !detectionRuleMetric.enabled - ? updatedUsage.elastic_total.disabled + 1 - : updatedUsage.elastic_total.disabled, - alerts: updatedUsage.elastic_total.alerts + detectionRuleMetric.alert_count_daily, - cases: updatedUsage.elastic_total.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? updatedUsage.elastic_total.legacy_notifications_enabled + 1 - : updatedUsage.elastic_total.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? updatedUsage.elastic_total.legacy_notifications_disabled + 1 - : updatedUsage.elastic_total.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? updatedUsage.elastic_total.notifications_enabled + 1 - : updatedUsage.elastic_total.notifications_enabled, - notifications_disabled: notificationDisabled - ? updatedUsage.elastic_total.notifications_disabled + 1 - : updatedUsage.elastic_total.notifications_disabled, - }, - }; - } else { - updatedUsage = { - ...updatedUsage, - custom_total: { - ...updatedUsage.custom_total, - enabled: detectionRuleMetric.enabled - ? updatedUsage.custom_total.enabled + 1 - : updatedUsage.custom_total.enabled, - disabled: !detectionRuleMetric.enabled - ? updatedUsage.custom_total.disabled + 1 - : updatedUsage.custom_total.disabled, - alerts: updatedUsage.custom_total.alerts + detectionRuleMetric.alert_count_daily, - cases: updatedUsage.custom_total.cases + detectionRuleMetric.cases_count_total, - legacy_notifications_enabled: legacyNotificationEnabled - ? updatedUsage.custom_total.legacy_notifications_enabled + 1 - : updatedUsage.custom_total.legacy_notifications_enabled, - legacy_notifications_disabled: legacyNotificationDisabled - ? updatedUsage.custom_total.legacy_notifications_disabled + 1 - : updatedUsage.custom_total.legacy_notifications_disabled, - notifications_enabled: notificationEnabled - ? updatedUsage.custom_total.notifications_enabled + 1 - : updatedUsage.custom_total.notifications_enabled, - notifications_disabled: notificationDisabled - ? updatedUsage.custom_total.notifications_disabled + 1 - : updatedUsage.custom_total.notifications_disabled, - }, - }; - } - - return updatedUsage; -}; - -const MAX_RESULTS_WINDOW = 10_000; // elasticsearch index.max_result_window default value - -export const getDetectionRuleMetrics = async ( - kibanaIndex: string, - signalsIndex: string, - esClient: ElasticsearchClient, - savedObjectClient: SavedObjectsClientContract -): Promise => { - let rulesUsage: DetectionRulesTypeUsage = initialDetectionRulesUsage; - const ruleSearchOptions: RuleSearchParams = { - body: { - query: { - bool: { - filter: { - terms: { - 'alert.alertTypeId': [ - SIGNALS_ID, - EQL_RULE_TYPE_ID, - ML_RULE_TYPE_ID, - QUERY_RULE_TYPE_ID, - SAVED_QUERY_RULE_TYPE_ID, - INDICATOR_RULE_TYPE_ID, - THRESHOLD_RULE_TYPE_ID, - ], - }, - }, - }, - }, - }, - filter_path: [], - ignore_unavailable: true, - index: kibanaIndex, - size: MAX_RESULTS_WINDOW, - }; - - try { - const ruleResults = await esClient.search(ruleSearchOptions); - const detectionAlertsResp = (await esClient.search({ - index: `${signalsIndex}*`, - size: MAX_RESULTS_WINDOW, - body: { - aggs: { - detectionAlerts: { - terms: { field: ALERT_RULE_UUID }, - }, - }, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: 'now-24h', - lte: 'now', - }, - }, - }, - ], - }, - }, - }, - })) as AlertsAggregationResponse; - - const cases = await savedObjectClient.find({ - type: CASE_COMMENT_SAVED_OBJECT, - page: 1, - perPage: MAX_RESULTS_WINDOW, - namespaces: ['*'], - filter: `${CASE_COMMENT_SAVED_OBJECT}.attributes.type: alert`, - }); - - // Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove this function. - const legacyRuleActions = - await savedObjectClient.find({ - type: legacyRuleActionsSavedObjectType, - page: 1, - perPage: MAX_RESULTS_WINDOW, - namespaces: ['*'], - }); - - const legacyNotificationRuleIds = legacyRuleActions.saved_objects.reduce( - (cache, legacyNotificationsObject) => { - const ruleRef = legacyNotificationsObject.references.find( - (reference) => reference.name === 'alert_0' && reference.type === 'alert' - ); - if (ruleRef != null) { - const enabled = legacyNotificationsObject.attributes.ruleThrottle !== 'no_actions'; - cache.set(ruleRef.id, { enabled }); - } - return cache; - }, - new Map() - ); - - const casesCache = cases.saved_objects.reduce((cache, { attributes: casesObject }) => { - const ruleId = casesObject.rule.id; - if (ruleId != null) { - const cacheCount = cache.get(ruleId); - if (cacheCount === undefined) { - cache.set(ruleId, 1); - } else { - cache.set(ruleId, cacheCount + 1); - } - } - return cache; - }, new Map()); - - const alertBuckets = detectionAlertsResp.aggregations?.detectionAlerts?.buckets ?? []; - - const alertsCache = new Map(); - alertBuckets.map((bucket) => alertsCache.set(bucket.key, bucket.doc_count)); - if (ruleResults.hits?.hits?.length > 0) { - const ruleObjects = ruleResults.hits.hits.map((hit) => { - const ruleId = hit._id.split(':')[1]; - const isElastic = isElasticRule(hit._source?.alert.tags); - - // Even if the legacy notification is set to "no_actions" we still count the rule as having a legacy notification that is not migrated yet. - const hasLegacyNotification = legacyNotificationRuleIds.get(ruleId) != null; - - // We only count a rule as having a notification and being "enabled" if it is _not_ set to "no_actions"/"muteAll" and it has at least one action within its array. - const hasNotification = - !hasLegacyNotification && - hit._source?.alert.actions != null && - hit._source?.alert.actions.length > 0 && - hit._source?.alert.muteAll !== true; - - return { - rule_name: hit._source?.alert.name, - rule_id: hit._source?.alert.params.ruleId, - rule_type: hit._source?.alert.params.type, - rule_version: Number(hit._source?.alert.params.version), - enabled: hit._source?.alert.enabled, - elastic_rule: isElastic, - created_on: hit._source?.alert.createdAt, - updated_on: hit._source?.alert.updatedAt, - alert_count_daily: alertsCache.get(ruleId) || 0, - cases_count_total: casesCache.get(ruleId) || 0, - has_legacy_notification: hasLegacyNotification, - has_notification: hasNotification, - } as DetectionRuleMetric; - }); - - // Only bring back rule detail on elastic prepackaged detection rules - const elasticRuleObjects = ruleObjects.filter((hit) => hit.elastic_rule === true); - - rulesUsage = ruleObjects.reduce((usage, rule) => { - return updateDetectionRuleUsage(rule, usage); - }, rulesUsage); - - return { - detection_rule_detail: elasticRuleObjects, - detection_rule_usage: rulesUsage, - }; - } - } catch (e) { - // ignore failure, usage will be zeroed - } - - return { - detection_rule_detail: [], - detection_rule_usage: rulesUsage, - }; -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts new file mode 100644 index 0000000000000..0d885aa3b142c --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_initial_usage.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DetectionMetrics } from './types'; + +import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; +import { getInitialRulesUsage } from './rules/get_initial_usage'; + +/** + * Initial detection metrics initialized. + */ +export const getInitialDetectionMetrics = (): DetectionMetrics => ({ + ml_jobs: { + ml_job_usage: getInitialMlJobUsage(), + ml_job_metrics: [], + }, + detection_rules: { + detection_rule_detail: [], + detection_rule_usage: getInitialRulesUsage(), + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/detections.test.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts similarity index 66% rename from x-pack/plugins/security_solution/server/usage/detections/detections.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts index 866fa226e2ecf..65929039bc104 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detections.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.test.ts @@ -5,48 +5,69 @@ * 2.0. */ +import type { DetectionMetrics } from './types'; + import { elasticsearchServiceMock, + loggingSystemMock, savedObjectsClientMock, } from '../../../../../../src/core/server/mocks'; import { mlServicesMock } from '../../lib/machine_learning/mocks'; -import { fetchDetectionsMetrics } from './index'; import { - getMockJobSummaryResponse, + getMockMlJobSummaryResponse, getMockListModulesResponse, getMockMlJobDetailsResponse, getMockMlJobStatsResponse, getMockMlDatafeedStatsResponse, getMockRuleSearchResponse, +} from './ml_jobs/get_metrics.mocks'; +import { getMockRuleAlertsResponse, - getMockAlertCasesResponse, -} from './detections.mocks'; -import { getInitialDetectionMetrics, initialDetectionRulesUsage } from './detection_rule_helpers'; -import { DetectionMetrics } from './types'; + getMockAlertCaseCommentsResponse, + getEmptySavedObjectResponse, +} from './rules/get_metrics.mocks'; +import { getInitialDetectionMetrics } from './get_initial_usage'; +import { getDetectionsMetrics } from './get_metrics'; +import { getInitialRulesUsage } from './rules/get_initial_usage'; describe('Detections Usage and Metrics', () => { - let esClientMock: ReturnType; - let mlMock: ReturnType; + let esClient: ReturnType; + let mlClient: ReturnType; let savedObjectsClient: ReturnType; - describe('getDetectionRuleMetrics()', () => { + describe('getRuleMetrics()', () => { beforeEach(() => { - esClientMock = elasticsearchServiceMock.createClusterClient().asInternalUser; - mlMock = mlServicesMock.createSetupContract(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mlClient = mlServicesMock.createSetupContract(); savedObjectsClient = savedObjectsClientMock.create(); }); it('returns zeroed counts if calls are empty', async () => { - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual(getInitialDetectionMetrics()); }); it('returns information with rule, alerts and cases', async () => { - esClientMock.search - .mockResponseOnce(getMockRuleSearchResponse()) - .mockResponseOnce(getMockRuleAlertsResponse(3400)); - savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + esClient.search.mockResponseOnce(getMockRuleAlertsResponse(3400)); + savedObjectsClient.find.mockResolvedValueOnce(getMockRuleSearchResponse()); + savedObjectsClient.find.mockResolvedValueOnce(getMockAlertCaseCommentsResponse()); + // Get empty saved object for legacy notification system. + savedObjectsClient.find.mockResolvedValueOnce(getEmptySavedObjectResponse()); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual({ ...getInitialDetectionMetrics(), @@ -68,7 +89,7 @@ describe('Detections Usage and Metrics', () => { }, ], detection_rule_usage: { - ...initialDetectionRulesUsage, + ...getInitialRulesUsage(), query: { enabled: 0, disabled: 1, @@ -95,18 +116,26 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with on non elastic prebuilt rule', async () => { - esClientMock.search - .mockResponseOnce(getMockRuleSearchResponse('not_immutable')) - .mockResponseOnce(getMockRuleAlertsResponse(800)); - savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + esClient.search.mockResponseOnce(getMockRuleAlertsResponse(800)); + savedObjectsClient.find.mockResolvedValueOnce(getMockRuleSearchResponse('not_immutable')); + savedObjectsClient.find.mockResolvedValueOnce(getMockAlertCaseCommentsResponse()); + // Get empty saved object for legacy notification system. + savedObjectsClient.find.mockResolvedValueOnce(getEmptySavedObjectResponse()); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual({ ...getInitialDetectionMetrics(), detection_rules: { detection_rule_detail: [], // *should not* contain custom detection rule details detection_rule_usage: { - ...initialDetectionRulesUsage, + ...getInitialRulesUsage(), custom_total: { alerts: 800, cases: 1, @@ -133,11 +162,20 @@ describe('Detections Usage and Metrics', () => { }); it('returns information with rule, no alerts and no cases', async () => { - esClientMock.search - .mockResponseOnce(getMockRuleSearchResponse()) - .mockResponseOnce(getMockRuleAlertsResponse(0)); - savedObjectsClient.find.mockResolvedValue(getMockAlertCasesResponse()); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + esClient.search.mockResponseOnce(getMockRuleAlertsResponse(0)); + savedObjectsClient.find.mockResolvedValueOnce(getMockRuleSearchResponse()); + savedObjectsClient.find.mockResolvedValueOnce(getMockAlertCaseCommentsResponse()); + // Get empty saved object for legacy notification system. + savedObjectsClient.find.mockResolvedValueOnce(getEmptySavedObjectResponse()); + + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual({ ...getInitialDetectionMetrics(), @@ -159,7 +197,7 @@ describe('Detections Usage and Metrics', () => { }, ], detection_rule_usage: { - ...initialDetectionRulesUsage, + ...getInitialRulesUsage(), elastic_total: { alerts: 0, cases: 1, @@ -186,29 +224,38 @@ describe('Detections Usage and Metrics', () => { }); }); - describe('fetchDetectionsMetrics()', () => { + describe('getDetectionsMetrics()', () => { beforeEach(() => { - esClientMock = elasticsearchServiceMock.createClusterClient().asInternalUser; - mlMock = mlServicesMock.createSetupContract(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mlClient = mlServicesMock.createSetupContract(); savedObjectsClient = savedObjectsClientMock.create(); + savedObjectsClient.find.mockResolvedValue(getEmptySavedObjectResponse()); }); it('returns an empty array if there is no data', async () => { - mlMock.anomalyDetectorsProvider.mockReturnValue({ + mlClient.anomalyDetectorsProvider.mockReturnValue({ jobs: null, jobStats: null, - } as unknown as ReturnType); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + } as unknown as ReturnType); + const logger = loggingSystemMock.createLogger(); + const result = await getDetectionsMetrics({ + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual(getInitialDetectionMetrics()); }); it('returns an ml job telemetry object from anomaly detectors provider', async () => { - const mockJobSummary = jest.fn().mockResolvedValue(getMockJobSummaryResponse()); + const logger = loggingSystemMock.createLogger(); + const mockJobSummary = jest.fn().mockResolvedValue(getMockMlJobSummaryResponse()); const mockListModules = jest.fn().mockResolvedValue(getMockListModulesResponse()); - mlMock.modulesProvider.mockReturnValue({ + mlClient.modulesProvider.mockReturnValue({ listModules: mockListModules, - } as unknown as ReturnType); - mlMock.jobServiceProvider.mockReturnValue({ + } as unknown as ReturnType); + mlClient.jobServiceProvider.mockReturnValue({ jobsSummary: mockJobSummary, }); const mockJobsResponse = jest.fn().mockResolvedValue(getMockMlJobDetailsResponse()); @@ -217,13 +264,19 @@ describe('Detections Usage and Metrics', () => { .fn() .mockResolvedValue(getMockMlDatafeedStatsResponse()); - mlMock.anomalyDetectorsProvider.mockReturnValue({ + mlClient.anomalyDetectorsProvider.mockReturnValue({ jobs: mockJobsResponse, jobStats: mockJobStatsResponse, datafeedStats: mockDatafeedStatsResponse, - } as unknown as ReturnType); + } as unknown as ReturnType); - const result = await fetchDetectionsMetrics('', '', esClientMock, savedObjectsClient, mlMock); + const result = await getDetectionsMetrics({ + signalsIndex: '', + esClient, + savedObjectsClient, + logger, + mlClient, + }); expect(result).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts new file mode 100644 index 0000000000000..258945fba662a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/get_metrics.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from 'kibana/server'; +import type { MlPluginSetup } from '../../../../ml/server'; +import type { DetectionMetrics } from './types'; + +import { getMlJobMetrics } from './ml_jobs/get_metrics'; +import { getRuleMetrics } from './rules/get_metrics'; +import { getInitialRulesUsage } from './rules/get_initial_usage'; +import { getInitialMlJobUsage } from './ml_jobs/get_initial_usage'; + +export interface GetDetectionsMetricsOptions { + signalsIndex: string; + esClient: ElasticsearchClient; + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; + mlClient: MlPluginSetup | undefined; +} + +export const getDetectionsMetrics = async ({ + signalsIndex, + esClient, + savedObjectsClient, + logger, + mlClient, +}: GetDetectionsMetricsOptions): Promise => { + const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([ + getMlJobMetrics({ mlClient, savedObjectsClient, logger }), + getRuleMetrics({ signalsIndex, esClient, savedObjectsClient, logger }), + ]); + + return { + ml_jobs: + mlJobMetrics.status === 'fulfilled' + ? mlJobMetrics.value + : { ml_job_metrics: [], ml_job_usage: getInitialMlJobUsage() }, + detection_rules: + detectionRuleMetrics.status === 'fulfilled' + ? detectionRuleMetrics.value + : { detection_rule_detail: [], detection_rule_usage: getInitialRulesUsage() }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/index.ts b/x-pack/plugins/security_solution/server/usage/detections/index.ts deleted file mode 100644 index a8d2ead83eec7..0000000000000 --- a/x-pack/plugins/security_solution/server/usage/detections/index.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient, SavedObjectsClientContract } from '../../../../../../src/core/server'; -import { MlPluginSetup } from '../../../../ml/server'; -import { getDetectionRuleMetrics, initialDetectionRulesUsage } from './detection_rule_helpers'; -import { getMlJobMetrics, initialMlJobsUsage } from './detection_ml_helpers'; -import { DetectionMetrics } from './types'; - -import { INTERNAL_IMMUTABLE_KEY } from '../../../common/constants'; - -export const isElasticRule = (tags: string[] = []) => - tags.includes(`${INTERNAL_IMMUTABLE_KEY}:true`); - -export const fetchDetectionsMetrics = async ( - kibanaIndex: string, - signalsIndex: string, - esClient: ElasticsearchClient, - soClient: SavedObjectsClientContract, - mlClient: MlPluginSetup | undefined -): Promise => { - const [mlJobMetrics, detectionRuleMetrics] = await Promise.allSettled([ - getMlJobMetrics(mlClient, soClient), - getDetectionRuleMetrics(kibanaIndex, signalsIndex, esClient, soClient), - ]); - - return { - ml_jobs: - mlJobMetrics.status === 'fulfilled' - ? mlJobMetrics.value - : { ml_job_metrics: [], ml_job_usage: initialMlJobsUsage }, - detection_rules: - detectionRuleMetrics.status === 'fulfilled' - ? detectionRuleMetrics.value - : { detection_rule_detail: [], detection_rule_usage: initialDetectionRulesUsage }, - }; -}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts new file mode 100644 index 0000000000000..6e3ab3124baf1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_initial_usage.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MlJobUsage } from './types'; + +/** + * Default ml job usage count + */ +export const getInitialMlJobUsage = (): MlJobUsage => ({ + custom: { + enabled: 0, + disabled: 0, + }, + elastic: { + enabled: 0, + disabled: 0, + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts similarity index 65% rename from x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts rename to x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts index e7c1384152c5a..a507a76e0c4f2 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.mocks.ts @@ -5,81 +5,8 @@ * 2.0. */ -import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; - -export const getMockJobSummaryResponse = () => [ - { - id: 'linux_anomalous_network_activity_ecs', - description: - 'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)', - groups: ['auditbeat', 'process', 'siem'], - processed_record_count: 141889, - memory_status: 'ok', - jobState: 'opened', - hasDatafeed: true, - datafeedId: 'datafeed-linux_anomalous_network_activity_ecs', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'started', - latestTimestampMs: 1594085401911, - earliestTimestampMs: 1593054845656, - latestResultsTimestampMs: 1594085401911, - isSingleMetricViewerJob: true, - nodeName: 'node', - }, - { - id: 'linux_anomalous_network_port_activity_ecs', - description: - 'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)', - groups: ['auditbeat', 'process', 'siem'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'closed', - hasDatafeed: true, - datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'stopped', - isSingleMetricViewerJob: true, - }, - { - id: 'other_job', - description: 'a job that is custom', - groups: ['auditbeat', 'process', 'security'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'closed', - hasDatafeed: true, - datafeedId: 'datafeed-other', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'stopped', - isSingleMetricViewerJob: true, - }, - { - id: 'another_job', - description: 'another job that is custom', - groups: ['auditbeat', 'process', 'security'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'opened', - hasDatafeed: true, - datafeedId: 'datafeed-another', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'started', - isSingleMetricViewerJob: true, - }, - { - id: 'irrelevant_job', - description: 'a non-security job', - groups: ['auditbeat', 'process'], - processed_record_count: 0, - memory_status: 'ok', - jobState: 'opened', - hasDatafeed: true, - datafeedId: 'datafeed-another', - datafeedIndices: ['auditbeat-*'], - datafeedState: 'started', - isSingleMetricViewerJob: true, - }, -]; +import type { SavedObjectsFindResponse } from 'kibana/server'; +import type { RuleSearchResult } from '../../types'; export const getMockListModulesResponse = () => [ { @@ -162,6 +89,80 @@ export const getMockListModulesResponse = () => [ }, ]; +export const getMockMlJobSummaryResponse = () => [ + { + id: 'linux_anomalous_network_activity_ecs', + description: + 'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)', + groups: ['auditbeat', 'process', 'siem'], + processed_record_count: 141889, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-linux_anomalous_network_activity_ecs', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + latestTimestampMs: 1594085401911, + earliestTimestampMs: 1593054845656, + latestResultsTimestampMs: 1594085401911, + isSingleMetricViewerJob: true, + nodeName: 'node', + }, + { + id: 'linux_anomalous_network_port_activity_ecs', + description: + 'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)', + groups: ['auditbeat', 'process', 'siem'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'closed', + hasDatafeed: true, + datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'stopped', + isSingleMetricViewerJob: true, + }, + { + id: 'other_job', + description: 'a job that is custom', + groups: ['auditbeat', 'process', 'security'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'closed', + hasDatafeed: true, + datafeedId: 'datafeed-other', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'stopped', + isSingleMetricViewerJob: true, + }, + { + id: 'another_job', + description: 'another job that is custom', + groups: ['auditbeat', 'process', 'security'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-another', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + isSingleMetricViewerJob: true, + }, + { + id: 'irrelevant_job', + description: 'a non-security job', + groups: ['auditbeat', 'process'], + processed_record_count: 0, + memory_status: 'ok', + jobState: 'opened', + hasDatafeed: true, + datafeedId: 'datafeed-another', + datafeedIndices: ['auditbeat-*'], + datafeedState: 'started', + isSingleMetricViewerJob: true, + }, +]; + export const getMockMlJobDetailsResponse = () => ({ count: 20, jobs: [ @@ -291,177 +292,100 @@ export const getMockMlDatafeedStatsResponse = () => ({ export const getMockRuleSearchResponse = ( immutableTag: string = '__internal_immutable:true' -): SearchResponse => ({ - took: 2, - timed_out: false, - _shards: { +): SavedObjectsFindResponse => + ({ + page: 1, + per_page: 1_000, total: 1, - successful: 1, - skipped: 0, - failed: 0, - }, - hits: { - total: { - value: 1093, - relation: 'eq', - }, - max_score: 0, - hits: [ + saved_objects: [ { - _index: '.kibanaindex', - _id: 'alert:6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', - _score: 0, - _source: { - alert: { - name: 'Azure Diagnostic Settings Deletion', - tags: [ - 'Elastic', - 'Cloud', - 'Azure', - 'Continuous Monitoring', - 'SecOps', - 'Monitoring', - '__internal_rule_id:5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', - `${immutableTag}`, + type: 'alert', + id: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + namespaces: ['default'], + attributes: { + name: 'Azure Diagnostic Settings Deletion', + tags: [ + 'Elastic', + 'Cloud', + 'Azure', + 'Continuous Monitoring', + 'SecOps', + 'Monitoring', + '__internal_rule_id:5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', + `${immutableTag}`, + ], + alertTypeId: 'siem.queryRule', + consumer: 'siem', + params: { + author: ['Elastic'], + description: + 'Identifies the deletion of diagnostic settings in Azure, which send platform logs and metrics to different destinations. An adversary may delete diagnostic settings in an attempt to evade defenses.', + ruleId: '5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', + index: ['filebeat-*', 'logs-azure*'], + falsePositives: [ + 'Deletion of diagnostic settings may be done by a system or network administrator. Verify whether the username, hostname, and/or resource name should be making changes in your environment. Diagnostic settings deletion from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule.', ], - alertTypeId: 'siem.signals', - consumer: 'siem', - params: { - author: ['Elastic'], - description: - 'Identifies the deletion of diagnostic settings in Azure, which send platform logs and metrics to different destinations. An adversary may delete diagnostic settings in an attempt to evade defenses.', - ruleId: '5370d4cd-2bb3-4d71-abf5-1e1d0ff5a2de', - index: ['filebeat-*', 'logs-azure*'], - falsePositives: [ - 'Deletion of diagnostic settings may be done by a system or network administrator. Verify whether the username, hostname, and/or resource name should be making changes in your environment. Diagnostic settings deletion from unfamiliar users or hosts should be investigated. If known behavior is causing false positives, it can be exempted from the rule.', - ], - from: 'now-25m', - immutable: true, - query: - 'event.dataset:azure.activitylogs and azure.activitylogs.operation_name:"MICROSOFT.INSIGHTS/DIAGNOSTICSETTINGS/DELETE" and event.outcome:(Success or success)', - language: 'kuery', - license: 'Elastic License v2', - outputIndex: '.siem-signals', - maxSignals: 100, - riskScore: 47, - timestampOverride: 'event.ingested', - to: 'now', - type: 'query', - references: [ - 'https://docs.microsoft.com/en-us/azure/azure-monitor/platform/diagnostic-settings', - ], - note: 'The Azure Filebeat module must be enabled to use this rule.', - version: 4, - exceptionsList: [], - }, - schedule: { - interval: '5m', - }, - enabled: false, - actions: [], - throttle: null, - notifyWhen: 'onActiveAlert', - apiKeyOwner: null, - apiKey: null, - createdBy: 'user', - updatedBy: 'user', - createdAt: '2021-03-23T17:15:59.634Z', - updatedAt: '2021-03-23T17:15:59.634Z', - muteAll: false, - mutedInstanceIds: [], - executionStatus: { - status: 'pending', - lastExecutionDate: '2021-03-23T17:15:59.634Z', - error: null, - }, - meta: { - versionApiKeyLastmodified: '8.0.0', + from: 'now-25m', + immutable: true, + query: + 'event.dataset:azure.activitylogs and azure.activitylogs.operation_name:"MICROSOFT.INSIGHTS/DIAGNOSTICSETTINGS/DELETE" and event.outcome:(Success or success)', + language: 'kuery', + license: 'Elastic License v2', + outputIndex: '.siem-signals', + maxSignals: 100, + riskScore: 47, + timestampOverride: 'event.ingested', + to: 'now', + type: 'query', + references: [ + 'https://docs.microsoft.com/en-us/azure/azure-monitor/platform/diagnostic-settings', + ], + note: 'The Azure Filebeat module must be enabled to use this rule.', + version: 4, + exceptionsList: [], + }, + schedule: { + interval: '5m', + }, + enabled: false, + actions: [], + throttle: null, + notifyWhen: 'onActiveAlert', + apiKeyOwner: null, + apiKey: '', + legacyId: null, + createdBy: 'user', + updatedBy: 'user', + createdAt: '2021-03-23T17:15:59.634Z', + updatedAt: '2021-03-23T17:15:59.634Z', + muteAll: true, + mutedInstanceIds: [], + monitoring: { + execution: { + history: [], + calculated_metrics: { + success_ratio: 1, + p99: 7981, + p50: 1653, + p95: 6523.699999999996, + }, }, }, - type: 'alert', - references: [], - migrationVersion: { - alert: '7.13.0', + meta: { + versionApiKeyLastmodified: '8.2.0', }, - coreMigrationVersion: '8.0.0', - updated_at: '2021-03-23T17:15:59.634Z', + scheduledTaskId: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', }, - }, - ], - }, -}); - -export const getMockRuleAlertsResponse = (docCount: number): SearchResponse => ({ - took: 7, - timed_out: false, - _shards: { - total: 1, - successful: 1, - skipped: 0, - failed: 0, - }, - hits: { - total: { - value: 7322, - relation: 'eq', - }, - max_score: null, - hits: [], - }, - aggregations: { - detectionAlerts: { - doc_count_error_upper_bound: 0, - sum_other_doc_count: 0, - buckets: [ - { - key: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', - doc_count: docCount, - }, - ], - }, - }, -}); - -export const getMockAlertCasesResponse = () => ({ - page: 1, - per_page: 10000, - total: 4, - saved_objects: [ - { - type: 'cases-comments', - id: '3bb5cc10-9249-11eb-85b7-254c8af1a983', - attributes: { - type: 'alert', - alertId: '54802763917f521249c9f68d0d4be0c26cc538404c26dfed1ae7dcfa94ea2226', - index: '.siem-signals-default-000001', - rule: { - id: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', - name: 'Azure Diagnostic Settings Deletion', - }, - created_at: '2021-03-31T17:47:59.449Z', - created_by: { - email: '', - full_name: '', - username: '', + references: [], + migrationVersion: { + alert: '8.0.0', }, - pushed_at: null, - pushed_by: null, - updated_at: null, - updated_by: null, + coreMigrationVersion: '8.2.0', + updated_at: '2021-03-23T17:15:59.634Z', + version: 'Wzk4NTQwLDNd', + score: 0, + sort: ['1644865254209', '19548'], }, - references: [ - { - type: 'cases', - name: 'associated-cases', - id: '3a3a4fa0-9249-11eb-85b7-254c8af1a983', - }, - ], - migrationVersion: {}, - coreMigrationVersion: '8.0.0', - updated_at: '2021-03-31T17:47:59.818Z', - version: 'WzI3MDIyODMsNF0=', - namespaces: ['default'], - score: 0, - }, - ], -}); + ], + // NOTE: We have to cast as "unknown" and then back to "RuleSearchResult" because "RuleSearchResult" isn't an exact type. See notes in the JSDocs fo that type. + } as unknown as SavedObjectsFindResponse); diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts new file mode 100644 index 0000000000000..2eea42f28d953 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/get_metrics.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRequest, SavedObjectsClientContract, Logger } from 'kibana/server'; +import type { MlDatafeedStats, MlJob, MlPluginSetup } from '../../../../../ml/server'; +import type { MlJobMetric, MlJobUsageMetric } from './types'; + +import { isJobStarted } from '../../../../common/machine_learning/helpers'; +import { isSecurityJob } from '../../../../common/machine_learning/is_security_job'; +import { getInitialMlJobUsage } from './get_initial_usage'; +import { updateMlJobUsage } from './update_usage'; +import { getJobCorrelations } from './transform_utils/get_job_correlations'; + +export interface GetMlJobMetricsOptions { + mlClient: MlPluginSetup | undefined; + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; +} + +export const getMlJobMetrics = async ({ + mlClient, + savedObjectsClient, + logger, +}: GetMlJobMetricsOptions): Promise => { + let jobsUsage = getInitialMlJobUsage(); + + if (mlClient == null) { + logger.debug( + 'Machine learning client is null/undefined, therefore not collecting telemetry from it' + ); + // early return if we don't have ml client + return { + ml_job_usage: getInitialMlJobUsage(), + ml_job_metrics: [], + }; + } + + try { + const fakeRequest = { headers: {} } as KibanaRequest; + + const modules = await mlClient.modulesProvider(fakeRequest, savedObjectsClient).listModules(); + const moduleJobs = modules.flatMap((module) => module.jobs); + const jobs = await mlClient.jobServiceProvider(fakeRequest, savedObjectsClient).jobsSummary(); + + jobsUsage = jobs.filter(isSecurityJob).reduce((usage, job) => { + const isElastic = moduleJobs.some((moduleJob) => moduleJob.id === job.id); + const isEnabled = isJobStarted(job.jobState, job.datafeedState); + + return updateMlJobUsage({ isElastic, isEnabled }, usage); + }, getInitialMlJobUsage()); + + const jobsType = 'security'; + const securityJobStats = await mlClient + .anomalyDetectorsProvider(fakeRequest, savedObjectsClient) + .jobStats(jobsType); + + const jobDetails = await mlClient + .anomalyDetectorsProvider(fakeRequest, savedObjectsClient) + .jobs(jobsType); + + const jobDetailsCache = new Map(); + jobDetails.jobs.forEach((detail) => jobDetailsCache.set(detail.job_id, detail)); + + const datafeedStats = await mlClient + .anomalyDetectorsProvider(fakeRequest, savedObjectsClient) + .datafeedStats(); + + const datafeedStatsCache = new Map(); + datafeedStats.datafeeds.forEach((datafeedStat) => + datafeedStatsCache.set(`${datafeedStat.datafeed_id}`, datafeedStat) + ); + + const jobMetrics = securityJobStats.jobs.map((stat) => { + const jobId = stat.job_id; + const jobDetail = jobDetailsCache.get(stat.job_id); + const datafeed = datafeedStatsCache.get(`datafeed-${jobId}`); + return getJobCorrelations({ stat, jobDetail, datafeed }); + }); + + return { + ml_job_usage: jobsUsage, + ml_job_metrics: jobMetrics, + }; + } catch (e) { + // ignore failure, usage will be zeroed. We don't log the message below as currently ML jobs when it does + // not have a "security" job will cause a throw. If this does not normally throw eventually on normal operations + // we should log a debug message like the following below to not unnecessarily worry users as this will not effect them: + // logger.debug( + // `Encountered unexpected condition in telemetry of message: ${e.message}, object: ${e}. Telemetry for "ml_jobs" will be skipped.` + // ); + return { + ml_job_usage: getInitialMlJobUsage(), + ml_job_metrics: [], + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts new file mode 100644 index 0000000000000..59a23c5dc7bd1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/transform_utils/get_job_correlations.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + MlDatafeedStats, + MlJob, + MlJobStats, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { MlJobMetric } from '../types'; + +export interface GetJobCorrelations { + stat: MlJobStats; + jobDetail: MlJob | undefined; + datafeed: MlDatafeedStats | undefined; +} + +export const getJobCorrelations = ({ + stat, + jobDetail, + datafeed, +}: GetJobCorrelations): MlJobMetric => { + return { + job_id: stat.job_id, + open_time: stat.open_time, + create_time: jobDetail?.create_time, + finished_time: jobDetail?.finished_time, + state: stat.state, + data_counts: { + bucket_count: stat.data_counts.bucket_count, + empty_bucket_count: stat.data_counts.empty_bucket_count, + input_bytes: stat.data_counts.input_bytes, + input_record_count: stat.data_counts.input_record_count, + last_data_time: stat.data_counts.last_data_time, + processed_record_count: stat.data_counts.processed_record_count, + }, + model_size_stats: { + bucket_allocation_failures_count: stat.model_size_stats.bucket_allocation_failures_count, + memory_status: stat.model_size_stats.memory_status, + model_bytes: stat.model_size_stats.model_bytes, + model_bytes_exceeded: stat.model_size_stats.model_bytes_exceeded, + model_bytes_memory_limit: stat.model_size_stats.model_bytes_memory_limit, + peak_model_bytes: stat.model_size_stats.peak_model_bytes, + }, + timing_stats: { + average_bucket_processing_time_ms: stat.timing_stats.average_bucket_processing_time_ms, + bucket_count: stat.timing_stats.bucket_count, + exponential_average_bucket_processing_time_ms: + stat.timing_stats.exponential_average_bucket_processing_time_ms, + exponential_average_bucket_processing_time_per_hour_ms: + stat.timing_stats.exponential_average_bucket_processing_time_per_hour_ms, + maximum_bucket_processing_time_ms: stat.timing_stats.maximum_bucket_processing_time_ms, + minimum_bucket_processing_time_ms: stat.timing_stats.minimum_bucket_processing_time_ms, + total_bucket_processing_time_ms: stat.timing_stats.total_bucket_processing_time_ms, + }, + datafeed: { + datafeed_id: datafeed?.datafeed_id, + state: datafeed?.state, + timing_stats: { + bucket_count: datafeed?.timing_stats.bucket_count, + exponential_average_search_time_per_hour_ms: + datafeed?.timing_stats.exponential_average_search_time_per_hour_ms, + search_count: datafeed?.timing_stats.search_count, + total_search_time_ms: datafeed?.timing_stats.total_search_time_ms, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts new file mode 100644 index 0000000000000..c50fc3166977a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/types.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + MlDataCounts, + MlDatafeedState, + MlDatafeedStats, + MlDatafeedTimingStats, + MlJob, + MlJobState, + MlJobStats, + MlJobTimingStats, + MlModelSizeStats, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +interface FeatureUsage { + enabled: number; + disabled: number; +} + +export interface MlJobUsage { + custom: FeatureUsage; + elastic: FeatureUsage; +} + +export interface MlJobMetric { + job_id: MlJobStats['job_id']; + create_time?: MlJob['create_time']; + finished_time?: MlJob['finished_time']; + open_time?: MlJobStats['open_time']; + state: MlJobState; + data_counts: Partial; + model_size_stats: Partial; + timing_stats: Partial; + datafeed: MlDataFeed; +} + +export interface MlJobUsageMetric { + ml_job_usage: MlJobUsage; + ml_job_metrics: MlJobMetric[]; +} + +export interface DetectionsMetric { + isElastic: boolean; + isEnabled: boolean; +} + +export interface MlDataFeed { + datafeed_id?: MlDatafeedStats['datafeed_id']; + state?: MlDatafeedState; + timing_stats: Partial; +} diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.test.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.test.ts similarity index 70% rename from x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.test.ts index 3ca0faeca7d36..9d0dc7c02e568 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_ml_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.test.ts @@ -5,16 +5,17 @@ * 2.0. */ -import { initialMlJobsUsage, updateMlJobsUsage } from './detection_ml_helpers'; +import { getInitialMlJobUsage } from './get_initial_usage'; +import { updateMlJobUsage } from './update_usage'; describe('Security Machine Learning usage metrics', () => { describe('Updates metrics with job information', () => { it('Should update ML total for elastic rules', async () => { - const initialUsage = initialMlJobsUsage; + const initialUsage = getInitialMlJobUsage(); const isElastic = true; const isEnabled = true; - const updatedUsage = updateMlJobsUsage({ isElastic, isEnabled }, initialUsage); + const updatedUsage = updateMlJobUsage({ isElastic, isEnabled }, initialUsage); expect(updatedUsage).toEqual( expect.objectContaining({ @@ -31,11 +32,11 @@ describe('Security Machine Learning usage metrics', () => { }); it('Should update ML total for custom rules', async () => { - const initialUsage = initialMlJobsUsage; + const initialUsage = getInitialMlJobUsage(); const isElastic = false; const isEnabled = true; - const updatedUsage = updateMlJobsUsage({ isElastic, isEnabled }, initialUsage); + const updatedUsage = updateMlJobUsage({ isElastic, isEnabled }, initialUsage); expect(updatedUsage).toEqual( expect.objectContaining({ @@ -52,10 +53,9 @@ describe('Security Machine Learning usage metrics', () => { }); it('Should update ML total for both elastic and custom rules', async () => { - const initialUsage = initialMlJobsUsage; - - let updatedUsage = updateMlJobsUsage({ isElastic: true, isEnabled: true }, initialUsage); - updatedUsage = updateMlJobsUsage({ isElastic: false, isEnabled: true }, updatedUsage); + const initialUsage = getInitialMlJobUsage(); + let updatedUsage = updateMlJobUsage({ isElastic: true, isEnabled: true }, initialUsage); + updatedUsage = updateMlJobUsage({ isElastic: false, isEnabled: true }, updatedUsage); expect(updatedUsage).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts new file mode 100644 index 0000000000000..2306bfa051a3b --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/ml_jobs/update_usage.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DetectionsMetric, MlJobUsage } from './types'; + +export const updateMlJobUsage = (jobMetric: DetectionsMetric, usage: MlJobUsage): MlJobUsage => { + const { isEnabled, isElastic } = jobMetric; + if (isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + enabled: usage.elastic.enabled + 1, + }, + }; + } else if (!isEnabled && isElastic) { + return { + ...usage, + elastic: { + ...usage.elastic, + disabled: usage.elastic.disabled + 1, + }, + }; + } else if (isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + enabled: usage.custom.enabled + 1, + }, + }; + } else if (!isEnabled && !isElastic) { + return { + ...usage, + custom: { + ...usage.custom, + disabled: usage.custom.disabled + 1, + }, + }; + } else { + return usage; + } +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts new file mode 100644 index 0000000000000..81ea7aec800e3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_initial_usage.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage } from './types'; + +/** + * Default detection rule usage count, split by type + elastic/custom + */ +export const getInitialRulesUsage = (): RulesTypeUsage => ({ + query: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + threshold: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + eql: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + machine_learning: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + threat_match: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + elastic_total: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, + custom_total: { + enabled: 0, + disabled: 0, + alerts: 0, + cases: 0, + legacy_notifications_enabled: 0, + legacy_notifications_disabled: 0, + notifications_enabled: 0, + notifications_disabled: 0, + }, +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts new file mode 100644 index 0000000000000..1801d5bd67782 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.mocks.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { SavedObjectsFindResponse } from 'kibana/server'; +import type { AlertAggs } from '../../types'; +import { CommentAttributes, CommentType } from '../../../../../cases/common/api/cases/comment'; + +export const getMockRuleAlertsResponse = (docCount: number): SearchResponse => ({ + took: 7, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 7322, + relation: 'eq', + }, + max_score: null, + hits: [], + }, + aggregations: { + buckets: { + after_key: { + detectionAlerts: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + }, + buckets: [ + { + key: { + detectionAlerts: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + }, + doc_count: docCount, + }, + ], + }, + }, +}); + +export const getMockAlertCaseCommentsResponse = (): SavedObjectsFindResponse< + Partial, + never +> => ({ + page: 1, + per_page: 10000, + total: 4, + saved_objects: [ + { + type: 'cases-comments', + id: '3bb5cc10-9249-11eb-85b7-254c8af1a983', + attributes: { + type: CommentType.alert, + alertId: '54802763917f521249c9f68d0d4be0c26cc538404c26dfed1ae7dcfa94ea2226', + index: '.siem-signals-default-000001', + rule: { + id: '6eecd8c2-8bfb-11eb-afbe-1b7a66309c6d', + name: 'Azure Diagnostic Settings Deletion', + }, + created_at: '2021-03-31T17:47:59.449Z', + created_by: { + email: '', + full_name: '', + username: '', + }, + pushed_at: null, + pushed_by: null, + updated_at: null, + updated_by: null, + }, + references: [ + { + type: 'cases', + name: 'associated-cases', + id: '3a3a4fa0-9249-11eb-85b7-254c8af1a983', + }, + ], + migrationVersion: {}, + coreMigrationVersion: '8.0.0', + updated_at: '2021-03-31T17:47:59.818Z', + version: 'WzI3MDIyODMsNF0=', + namespaces: ['default'], + score: 0, + }, + ], +}); + +export const getEmptySavedObjectResponse = (): SavedObjectsFindResponse => ({ + page: 1, + per_page: 1_000, + total: 0, + saved_objects: [], +}); diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts new file mode 100644 index 0000000000000..b202ea964301c --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/get_metrics.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract, Logger } from 'kibana/server'; +import type { RuleAdoption } from './types'; + +import { updateRuleUsage } from './update_usage'; +import { getDetectionRules } from '../../queries/get_detection_rules'; +import { getAlerts } from '../../queries/get_alerts'; +import { MAX_PER_PAGE, MAX_RESULTS_WINDOW } from '../../constants'; +import { getInitialRulesUsage } from './get_initial_usage'; +import { getCaseComments } from '../../queries/get_case_comments'; +import { getRuleIdToCasesMap } from './transform_utils/get_rule_id_to_cases_map'; +import { getAlertIdToCountMap } from './transform_utils/get_alert_id_to_count_map'; +import { getRuleIdToEnabledMap } from './transform_utils/get_rule_id_to_enabled_map'; +import { getRuleObjectCorrelations } from './transform_utils/get_rule_object_correlations'; + +// eslint-disable-next-line no-restricted-imports +import { legacyGetRuleActions } from '../../queries/legacy_get_rule_actions'; + +export interface GetRuleMetricsOptions { + signalsIndex: string; + esClient: ElasticsearchClient; + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; +} + +export const getRuleMetrics = async ({ + signalsIndex, + esClient, + savedObjectsClient, + logger, +}: GetRuleMetricsOptions): Promise => { + try { + // gets rule saved objects + const ruleResults = await getDetectionRules({ + savedObjectsClient, + maxPerPage: MAX_PER_PAGE, + maxSize: MAX_RESULTS_WINDOW, + logger, + }); + + // early return if we don't have any detection rules then there is no need to query anything else + if (ruleResults.length === 0) { + return { + detection_rule_detail: [], + detection_rule_usage: getInitialRulesUsage(), + }; + } + + // gets the alerts data objects + const detectionAlertsRespPromise = getAlerts({ + esClient, + signalsIndex: `${signalsIndex}*`, + maxPerPage: MAX_PER_PAGE, + maxSize: MAX_RESULTS_WINDOW, + logger, + }); + + // gets cases saved objects + const caseCommentsPromise = getCaseComments({ + savedObjectsClient, + maxSize: MAX_PER_PAGE, + maxPerPage: MAX_RESULTS_WINDOW, + logger, + }); + + // gets the legacy rule actions to track legacy notifications. + const legacyRuleActionsPromise = legacyGetRuleActions({ + savedObjectsClient, + maxSize: MAX_PER_PAGE, + maxPerPage: MAX_RESULTS_WINDOW, + logger, + }); + + const [detectionAlertsResp, caseComments, legacyRuleActions] = await Promise.all([ + detectionAlertsRespPromise, + caseCommentsPromise, + legacyRuleActionsPromise, + ]); + + // create in-memory maps for correlation + const legacyNotificationRuleIds = getRuleIdToEnabledMap(legacyRuleActions); + const casesRuleIds = getRuleIdToCasesMap(caseComments); + const alertsCounts = getAlertIdToCountMap(detectionAlertsResp); + + // correlate the rule objects to the results + const rulesCorrelated = getRuleObjectCorrelations({ + ruleResults, + legacyNotificationRuleIds, + casesRuleIds, + alertsCounts, + }); + + // Only bring back rule detail on elastic prepackaged detection rules + const elasticRuleObjects = rulesCorrelated.filter((hit) => hit.elastic_rule === true); + + // calculate the rule usage + const rulesUsage = rulesCorrelated.reduce( + (usage, rule) => updateRuleUsage(rule, usage), + getInitialRulesUsage() + ); + + return { + detection_rule_detail: elasticRuleObjects, + detection_rule_usage: rulesUsage, + }; + } catch (e) { + // ignore failure, usage will be zeroed. We use debug mode to not unnecessarily worry users as this will not effect them. + logger.debug( + `Encountered unexpected condition in telemetry of message: ${e.message}, object: ${e}. Telemetry for "detection rules" being skipped.` + ); + return { + detection_rule_detail: [], + detection_rule_usage: getInitialRulesUsage(), + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts new file mode 100644 index 0000000000000..ce569564273e4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_alert_id_to_count_map.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AlertBucket } from '../../../types'; + +export const getAlertIdToCountMap = (alerts: AlertBucket[]): Map => { + const alertsCache = new Map(); + alerts.map((bucket) => alertsCache.set(bucket.key.detectionAlerts, bucket.doc_count)); + return alertsCache; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts new file mode 100644 index 0000000000000..d7ce790be0750 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_cases_map.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from 'kibana/server'; +import type { CommentAttributes } from '../../../../../../cases/common/api/cases/comment'; + +export const getRuleIdToCasesMap = ( + cases: Array> +): Map => { + return cases.reduce((cache, { attributes: casesObject }) => { + if (casesObject.type === 'alert') { + const ruleId = casesObject.rule.id; + if (ruleId != null) { + const cacheCount = cache.get(ruleId); + if (cacheCount === undefined) { + cache.set(ruleId, 1); + } else { + cache.set(ruleId, cacheCount + 1); + } + } + return cache; + } else { + return cache; + } + }, new Map()); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts new file mode 100644 index 0000000000000..b280d3a4ba17d --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_id_to_enabled_map.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from 'kibana/server'; +// eslint-disable-next-line no-restricted-imports +import type { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../../../lib/detection_engine/rule_actions/legacy_types'; + +export const getRuleIdToEnabledMap = ( + legacyRuleActions: Array< + SavedObjectsFindResult + > +): Map< + string, + { + enabled: boolean; + } +> => { + return legacyRuleActions.reduce((cache, legacyNotificationsObject) => { + const ruleRef = legacyNotificationsObject.references.find( + (reference) => reference.name === 'alert_0' && reference.type === 'alert' + ); + if (ruleRef != null) { + const enabled = legacyNotificationsObject.attributes.ruleThrottle !== 'no_actions'; + cache.set(ruleRef.id, { enabled }); + } + return cache; + }, new Map()); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts new file mode 100644 index 0000000000000..0c364efe73bd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/transform_utils/get_rule_object_correlations.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsFindResult } from 'kibana/server'; +import type { RuleMetric } from '../types'; +import type { RuleSearchResult } from '../../../types'; + +import { isElasticRule } from '../../../queries/utils/is_elastic_rule'; + +export interface RuleObjectCorrelationsOptions { + ruleResults: Array>; + legacyNotificationRuleIds: Map< + string, + { + enabled: boolean; + } + >; + casesRuleIds: Map; + alertsCounts: Map; +} + +export const getRuleObjectCorrelations = ({ + ruleResults, + legacyNotificationRuleIds, + casesRuleIds, + alertsCounts, +}: RuleObjectCorrelationsOptions): RuleMetric[] => { + return ruleResults.map((result) => { + const ruleId = result.id; + const { attributes } = result; + const isElastic = isElasticRule(attributes.tags); + + // Even if the legacy notification is set to "no_actions" we still count the rule as having a legacy notification that is not migrated yet. + const hasLegacyNotification = legacyNotificationRuleIds.get(ruleId) != null; + + // We only count a rule as having a notification and being "enabled" if it is _not_ set to "no_actions"/"muteAll" and it has at least one action within its array. + const hasNotification = + !hasLegacyNotification && + attributes.actions != null && + attributes.actions.length > 0 && + attributes.muteAll !== true; + + return { + rule_name: attributes.name, + rule_id: attributes.params.ruleId, + rule_type: attributes.params.type, + rule_version: attributes.params.version, + enabled: attributes.enabled, + elastic_rule: isElastic, + created_on: attributes.createdAt, + updated_on: attributes.updatedAt, + alert_count_daily: alertsCounts.get(ruleId) || 0, + cases_count_total: casesRuleIds.get(ruleId) || 0, + has_legacy_notification: hasLegacyNotification, + has_notification: hasNotification, + }; + }); +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/types.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/types.ts new file mode 100644 index 0000000000000..54b3e6d6a0084 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/types.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface FeatureTypeUsage { + enabled: number; + disabled: number; + alerts: number; + cases: number; + legacy_notifications_enabled: number; + legacy_notifications_disabled: number; + notifications_enabled: number; + notifications_disabled: number; +} + +export interface RulesTypeUsage { + query: FeatureTypeUsage; + threshold: FeatureTypeUsage; + eql: FeatureTypeUsage; + machine_learning: FeatureTypeUsage; + threat_match: FeatureTypeUsage; + elastic_total: FeatureTypeUsage; + custom_total: FeatureTypeUsage; +} + +export interface RuleAdoption { + detection_rule_detail: RuleMetric[]; + detection_rule_usage: RulesTypeUsage; +} + +export interface RuleMetric { + rule_name: string; + rule_id: string; + rule_type: string; + rule_version: number; + enabled: boolean; + elastic_rule: boolean; + created_on: string; + updated_on: string; + alert_count_daily: number; + cases_count_total: number; + has_legacy_notification: boolean; + has_notification: boolean; +} diff --git a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.test.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.test.ts similarity index 92% rename from x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.test.ts rename to x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.test.ts index c19e7b18f9e72..d878d0a5145ab 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detection_rule_helpers.test.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.test.ts @@ -5,8 +5,9 @@ * 2.0. */ -import { initialDetectionRulesUsage, updateDetectionRuleUsage } from './detection_rule_helpers'; -import { DetectionRuleMetric, DetectionRulesTypeUsage } from './types'; +import type { RuleMetric, RulesTypeUsage } from './types'; +import { updateRuleUsage } from './update_usage'; +import { getInitialRulesUsage } from './get_initial_usage'; interface StubRuleOptions { ruleType: string; @@ -26,7 +27,7 @@ const createStubRule = ({ caseCount, hasLegacyNotification, hasNotification, -}: StubRuleOptions): DetectionRuleMetric => ({ +}: StubRuleOptions): RuleMetric => ({ rule_name: 'rule-name', rule_id: 'id-123', rule_type: ruleType, @@ -53,10 +54,10 @@ describe('Detections Usage and Metrics', () => { hasLegacyNotification: false, hasNotification: false, }); - const usage = updateDetectionRuleUsage(stubRule, initialDetectionRulesUsage); + const usage = updateRuleUsage(stubRule, getInitialRulesUsage()); - expect(usage).toEqual({ - ...initialDetectionRulesUsage, + expect(usage).toEqual({ + ...getInitialRulesUsage(), elastic_total: { alerts: 1, cases: 1, @@ -127,14 +128,14 @@ describe('Detections Usage and Metrics', () => { hasNotification: false, }); - let usage = updateDetectionRuleUsage(stubEqlRule, initialDetectionRulesUsage); - usage = updateDetectionRuleUsage(stubQueryRuleOne, usage); - usage = updateDetectionRuleUsage(stubQueryRuleTwo, usage); - usage = updateDetectionRuleUsage(stubMachineLearningOne, usage); - usage = updateDetectionRuleUsage(stubMachineLearningTwo, usage); + let usage = updateRuleUsage(stubEqlRule, getInitialRulesUsage()); + usage = updateRuleUsage(stubQueryRuleOne, usage); + usage = updateRuleUsage(stubQueryRuleTwo, usage); + usage = updateRuleUsage(stubMachineLearningOne, usage); + usage = updateRuleUsage(stubMachineLearningTwo, usage); - expect(usage).toEqual({ - ...initialDetectionRulesUsage, + expect(usage).toEqual({ + ...getInitialRulesUsage(), custom_total: { alerts: 5, cases: 12, @@ -242,8 +243,8 @@ describe('Detections Usage and Metrics', () => { alertCount: 0, caseCount: 0, }); - const usage = updateDetectionRuleUsage(rule1, initialDetectionRulesUsage) as ReturnType< - typeof updateDetectionRuleUsage + const usage = updateRuleUsage(rule1, getInitialRulesUsage()) as ReturnType< + typeof updateRuleUsage > & { [key: string]: unknown }; expect(usage[ruleType]).toEqual( expect.objectContaining({ @@ -264,8 +265,8 @@ describe('Detections Usage and Metrics', () => { alertCount: 0, caseCount: 0, }); - const usageAddedByOne = updateDetectionRuleUsage(rule2, usage) as ReturnType< - typeof updateDetectionRuleUsage + const usageAddedByOne = updateRuleUsage(rule2, usage) as ReturnType< + typeof updateRuleUsage > & { [key: string]: unknown }; expect(usageAddedByOne[ruleType]).toEqual( diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts new file mode 100644 index 0000000000000..3aa3c3bbc29b0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/update_usage.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage, RuleMetric } from './types'; +import { updateQueryUsage } from './usage_utils/update_query_usage'; +import { updateTotalUsage } from './usage_utils/update_total_usage'; + +export const updateRuleUsage = ( + detectionRuleMetric: RuleMetric, + usage: RulesTypeUsage +): RulesTypeUsage => { + let updatedUsage = usage; + if (detectionRuleMetric.rule_type === 'query') { + updatedUsage = { + ...usage, + query: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'threshold') { + updatedUsage = { + ...usage, + threshold: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'eql') { + updatedUsage = { + ...usage, + eql: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'machine_learning') { + updatedUsage = { + ...usage, + machine_learning: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } else if (detectionRuleMetric.rule_type === 'threat_match') { + updatedUsage = { + ...usage, + threat_match: updateQueryUsage({ + ruleType: detectionRuleMetric.rule_type, + usage, + detectionRuleMetric, + }), + }; + } + + if (detectionRuleMetric.elastic_rule) { + updatedUsage = { + ...updatedUsage, + elastic_total: updateTotalUsage({ + detectionRuleMetric, + updatedUsage, + totalType: 'elastic_total', + }), + }; + } else { + updatedUsage = { + ...updatedUsage, + custom_total: updateTotalUsage({ + detectionRuleMetric, + updatedUsage, + totalType: 'custom_total', + }), + }; + } + + return updatedUsage; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts new file mode 100644 index 0000000000000..aae3f3fe00d0f --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/get_notifications_enabled_disabled.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleMetric } from '../types'; + +export const getNotificationsEnabledDisabled = ( + detectionRuleMetric: RuleMetric +): { + legacyNotificationEnabled: boolean; + legacyNotificationDisabled: boolean; + notificationEnabled: boolean; + notificationDisabled: boolean; +} => { + return { + legacyNotificationEnabled: + detectionRuleMetric.has_legacy_notification && detectionRuleMetric.enabled, + legacyNotificationDisabled: + detectionRuleMetric.has_legacy_notification && !detectionRuleMetric.enabled, + notificationEnabled: detectionRuleMetric.has_notification && detectionRuleMetric.enabled, + notificationDisabled: detectionRuleMetric.has_notification && !detectionRuleMetric.enabled, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts new file mode 100644 index 0000000000000..7f40ceec21c8a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_query_usage.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage, RuleMetric, FeatureTypeUsage } from '../types'; +import { getNotificationsEnabledDisabled } from './get_notifications_enabled_disabled'; + +export interface UpdateQueryUsageOptions { + ruleType: keyof RulesTypeUsage; + usage: RulesTypeUsage; + detectionRuleMetric: RuleMetric; +} + +export const updateQueryUsage = ({ + ruleType, + usage, + detectionRuleMetric, +}: UpdateQueryUsageOptions): FeatureTypeUsage => { + const { + legacyNotificationEnabled, + legacyNotificationDisabled, + notificationEnabled, + notificationDisabled, + } = getNotificationsEnabledDisabled(detectionRuleMetric); + return { + enabled: detectionRuleMetric.enabled ? usage[ruleType].enabled + 1 : usage[ruleType].enabled, + disabled: !detectionRuleMetric.enabled + ? usage[ruleType].disabled + 1 + : usage[ruleType].disabled, + alerts: usage[ruleType].alerts + detectionRuleMetric.alert_count_daily, + cases: usage[ruleType].cases + detectionRuleMetric.cases_count_total, + legacy_notifications_enabled: legacyNotificationEnabled + ? usage[ruleType].legacy_notifications_enabled + 1 + : usage[ruleType].legacy_notifications_enabled, + legacy_notifications_disabled: legacyNotificationDisabled + ? usage[ruleType].legacy_notifications_disabled + 1 + : usage[ruleType].legacy_notifications_disabled, + notifications_enabled: notificationEnabled + ? usage[ruleType].notifications_enabled + 1 + : usage[ruleType].notifications_enabled, + notifications_disabled: notificationDisabled + ? usage[ruleType].notifications_disabled + 1 + : usage[ruleType].notifications_disabled, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts new file mode 100644 index 0000000000000..ed0ff37e2a328 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/detections/rules/usage_utils/update_total_usage.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesTypeUsage, RuleMetric, FeatureTypeUsage } from '../types'; +import { getNotificationsEnabledDisabled } from './get_notifications_enabled_disabled'; + +export interface UpdateTotalUsageOptions { + detectionRuleMetric: RuleMetric; + updatedUsage: RulesTypeUsage; + totalType: 'custom_total' | 'elastic_total'; +} + +export const updateTotalUsage = ({ + detectionRuleMetric, + updatedUsage, + totalType, +}: UpdateTotalUsageOptions): FeatureTypeUsage => { + const { + legacyNotificationEnabled, + legacyNotificationDisabled, + notificationEnabled, + notificationDisabled, + } = getNotificationsEnabledDisabled(detectionRuleMetric); + + return { + enabled: detectionRuleMetric.enabled + ? updatedUsage[totalType].enabled + 1 + : updatedUsage[totalType].enabled, + disabled: !detectionRuleMetric.enabled + ? updatedUsage[totalType].disabled + 1 + : updatedUsage[totalType].disabled, + alerts: updatedUsage[totalType].alerts + detectionRuleMetric.alert_count_daily, + cases: updatedUsage[totalType].cases + detectionRuleMetric.cases_count_total, + legacy_notifications_enabled: legacyNotificationEnabled + ? updatedUsage[totalType].legacy_notifications_enabled + 1 + : updatedUsage[totalType].legacy_notifications_enabled, + legacy_notifications_disabled: legacyNotificationDisabled + ? updatedUsage[totalType].legacy_notifications_disabled + 1 + : updatedUsage[totalType].legacy_notifications_disabled, + notifications_enabled: notificationEnabled + ? updatedUsage[totalType].notifications_enabled + 1 + : updatedUsage[totalType].notifications_enabled, + notifications_disabled: notificationDisabled + ? updatedUsage[totalType].notifications_disabled + 1 + : updatedUsage[totalType].notifications_disabled, + }; +}; diff --git a/x-pack/plugins/security_solution/server/usage/detections/types.ts b/x-pack/plugins/security_solution/server/usage/detections/types.ts index a7eb4c387d4ba..2895e5c6f8b9a 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/types.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/types.ts @@ -5,165 +5,10 @@ * 2.0. */ -interface RuleSearchBody { - query: { - bool: { - filter: { - terms: { [key: string]: string[] }; - }; - }; - }; -} - -export interface RuleSearchParams { - body: RuleSearchBody; - filter_path: string[]; - ignore_unavailable: boolean; - index: string; - size: number; -} - -export interface RuleSearchResult { - alert: { - name: string; - enabled: boolean; - tags: string[]; - createdAt: string; - updatedAt: string; - muteAll: boolean | undefined | null; - params: DetectionRuleParms; - actions: unknown[]; - }; -} - -export interface DetectionsMetric { - isElastic: boolean; - isEnabled: boolean; -} - -interface DetectionRuleParms { - ruleId: string; - version: string; - type: string; -} - -interface FeatureUsage { - enabled: number; - disabled: number; -} - -interface FeatureTypeUsage { - enabled: number; - disabled: number; - alerts: number; - cases: number; - legacy_notifications_enabled: number; - legacy_notifications_disabled: number; - notifications_enabled: number; - notifications_disabled: number; -} -export interface DetectionRulesTypeUsage { - query: FeatureTypeUsage; - threshold: FeatureTypeUsage; - eql: FeatureTypeUsage; - machine_learning: FeatureTypeUsage; - threat_match: FeatureTypeUsage; - elastic_total: FeatureTypeUsage; - custom_total: FeatureTypeUsage; -} - -export interface MlJobsUsage { - custom: FeatureUsage; - elastic: FeatureUsage; -} - -export interface DetectionsUsage { - ml_jobs: MlJobsUsage; -} +import type { MlJobUsageMetric } from './ml_jobs/types'; +import type { RuleAdoption } from './rules/types'; export interface DetectionMetrics { - ml_jobs: MlJobUsage; - detection_rules: DetectionRuleAdoption; -} - -export interface MlJobDataCount { - bucket_count: number; - empty_bucket_count: number; - input_bytes: number; - input_record_count: number; - last_data_time: number; - processed_record_count: number; -} - -export interface MlJobModelSize { - bucket_allocation_failures_count: number; - memory_status: string; - model_bytes: number; - model_bytes_exceeded: number; - model_bytes_memory_limit: number; - peak_model_bytes: number; -} - -export interface MlTimingStats { - bucket_count: number; - exponential_average_bucket_processing_time_ms: number; - exponential_average_bucket_processing_time_per_hour_ms: number; - maximum_bucket_processing_time_ms: number; - minimum_bucket_processing_time_ms: number; - total_bucket_processing_time_ms: number; -} - -export interface MlJobMetric { - job_id: string; - open_time: string; - state: string; - data_counts: MlJobDataCount; - model_size_stats: MlJobModelSize; - timing_stats: MlTimingStats; -} - -export interface DetectionRuleMetric { - rule_name: string; - rule_id: string; - rule_type: string; - rule_version: number; - enabled: boolean; - elastic_rule: boolean; - created_on: string; - updated_on: string; - alert_count_daily: number; - cases_count_total: number; - has_legacy_notification: boolean; - has_notification: boolean; -} - -export interface AlertsAggregationResponse { - hits: { - total: { value: number }; - }; - aggregations: { - [aggName: string]: { - buckets: Array<{ key: string; doc_count: number }>; - }; - }; -} - -export interface CasesSavedObject { - type: string; - alertId: string; - index: string; - rule: { - id: string | null; - name: string | null; - }; -} - -export interface MlJobUsage { - ml_job_usage: MlJobsUsage; - ml_job_metrics: MlJobMetric[]; -} - -export interface DetectionRuleAdoption { - detection_rule_detail: DetectionRuleMetric[]; - detection_rule_usage: DetectionRulesTypeUsage; + ml_jobs: MlJobUsageMetric; + detection_rules: RuleAdoption; } diff --git a/x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts b/x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts new file mode 100644 index 0000000000000..aea462ecf1fa6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/get_internal_saved_objects_client.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup, SavedObjectsClientContract } from 'kibana/server'; + +import { SAVED_OBJECT_TYPES } from '../../../cases/common/constants'; +// eslint-disable-next-line no-restricted-imports +import { legacyRuleActionsSavedObjectType } from '../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; + +export async function getInternalSavedObjectsClient( + core: CoreSetup +): Promise { + return core.getStartServices().then(async ([coreStart]) => { + // note: we include the "cases" and "alert" hidden types here otherwise we would not be able to query them. If at some point cases and alert is not considered a hidden type this can be removed + return coreStart.savedObjects.createInternalRepository([ + 'alert', + legacyRuleActionsSavedObjectType, + ...SAVED_OBJECT_TYPES, + ]) as unknown as SavedObjectsClientContract; + }); +} diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts new file mode 100644 index 0000000000000..792ca28dcfba3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/get_alerts.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { OpenPointInTimeResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + AggregationsCompositeAggregation, + SearchRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; +import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; +import type { AlertBucket, AlertAggs } from '../types'; + +export interface GetAlertsOptions { + esClient: ElasticsearchClient; + signalsIndex: string; + maxSize: number; + maxPerPage: number; + logger: Logger; +} + +export const getAlerts = async ({ + esClient, + signalsIndex, + maxSize, + maxPerPage, + logger, +}: GetAlertsOptions): Promise => { + // default is from looking at Kibana saved objects and online documentation + const keepAlive = '5m'; + + // create and assign an initial point in time + let pitId: OpenPointInTimeResponse['id'] = ( + await esClient.openPointInTime({ + index: signalsIndex, + keep_alive: keepAlive, + }) + ).id; + + let after: AggregationsCompositeAggregation['after']; + let buckets: AlertBucket[] = []; + let fetchMore = true; + while (fetchMore) { + const ruleSearchOptions: SearchRequest = { + aggs: { + buckets: { + composite: { + size: Math.min(maxPerPage, maxSize - buckets.length), + sources: [ + { + detectionAlerts: { + terms: { + field: ALERT_RULE_UUID, + }, + }, + }, + ], + after, + }, + }, + }, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: 'now-24h', + lte: 'now', + }, + }, + }, + ], + }, + }, + track_total_hits: false, + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // TODO: Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + pit: { id: pitId }, + size: 0, + }; + logger.debug( + `Getting alerts with point in time (PIT) query: ${JSON.stringify(ruleSearchOptions)}` + ); + const body = await esClient.search(ruleSearchOptions); + if (body.aggregations?.buckets?.buckets != null) { + buckets = [...buckets, ...body.aggregations.buckets.buckets]; + } + if (body.aggregations?.buckets?.after_key != null) { + after = { + detectionAlerts: body.aggregations.buckets.after_key.detectionAlerts, + }; + } + + fetchMore = + body.aggregations?.buckets?.buckets != null && + body.aggregations?.buckets?.buckets.length !== 0 && + buckets.length < maxSize; + if (body.pit_id != null) { + pitId = body.pit_id; + } + } + try { + await esClient.closePointInTime({ id: pitId }); + } catch (error) { + // Don't fail due to a bad point in time closure. We have seen failures in e2e tests during nominal operations. + logger.warn( + `Error trying to close point in time: "${pitId}", it will expire within "${keepAlive}". Error is: "${error}"` + ); + } + logger.debug(`Returning alerts response of length: "${buckets.length}"`); + return buckets; +}; diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts b/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts new file mode 100644 index 0000000000000..0a6c7f2fc209a --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/get_case_comments.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsFindResult, + Logger, + SavedObjectsCreatePointInTimeFinderOptions, +} from 'kibana/server'; + +import { CASE_COMMENT_SAVED_OBJECT } from '../../../../cases/common/constants'; +import type { CommentAttributes } from '../../../../cases/common/api/cases/comment'; + +export interface GetCasesOptions { + savedObjectsClient: SavedObjectsClientContract; + maxSize: number; + maxPerPage: number; + logger: Logger; +} + +export const getCaseComments = async ({ + savedObjectsClient, + maxSize, + maxPerPage, + logger, +}: GetCasesOptions): Promise>> => { + const query: SavedObjectsCreatePointInTimeFinderOptions = { + type: CASE_COMMENT_SAVED_OBJECT, + perPage: maxPerPage, + namespaces: ['*'], + filter: `${CASE_COMMENT_SAVED_OBJECT}.attributes.type: alert`, + }; + logger.debug(`Getting cases with point in time (PIT) query:', ${JSON.stringify(query)}`); + const finder = savedObjectsClient.createPointInTimeFinder(query); + let responses: Array> = []; + for await (const response of finder.find()) { + const extra = responses.length + response.saved_objects.length - maxSize; + if (extra > 0) { + responses = [ + ...responses, + ...response.saved_objects.slice(-response.saved_objects.length, -extra), + ]; + } else { + responses = [...responses, ...response.saved_objects]; + } + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + + logger.debug(`Returning cases response of length: "${responses.length}"`); + return responses; +}; diff --git a/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts new file mode 100644 index 0000000000000..62f5691f73d07 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/get_detection_rules.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + Logger, + SavedObjectsCreatePointInTimeFinderOptions, + SavedObjectsClientContract, + SavedObjectsFindResult, +} from 'kibana/server'; +import { + SIGNALS_ID, + EQL_RULE_TYPE_ID, + INDICATOR_RULE_TYPE_ID, + ML_RULE_TYPE_ID, + QUERY_RULE_TYPE_ID, + THRESHOLD_RULE_TYPE_ID, + SAVED_QUERY_RULE_TYPE_ID, +} from '@kbn/securitysolution-rules'; +import type { RuleSearchResult } from '../types'; + +export interface GetDetectionRulesOptions { + maxSize: number; + maxPerPage: number; + logger: Logger; + savedObjectsClient: SavedObjectsClientContract; +} + +export const getDetectionRules = async ({ + maxSize, + maxPerPage, + logger, + savedObjectsClient, +}: GetDetectionRulesOptions): Promise>> => { + const filterAttribute = 'alert.attributes.alertTypeId'; + const filter = [ + `${filterAttribute}: ${SIGNALS_ID}`, + `${filterAttribute}: ${EQL_RULE_TYPE_ID}`, + `${filterAttribute}: ${ML_RULE_TYPE_ID}`, + `${filterAttribute}: ${QUERY_RULE_TYPE_ID}`, + `${filterAttribute}: ${SAVED_QUERY_RULE_TYPE_ID}`, + `${filterAttribute}: ${THRESHOLD_RULE_TYPE_ID}`, + `${filterAttribute}: ${INDICATOR_RULE_TYPE_ID}`, + ].join(' OR '); + + const query: SavedObjectsCreatePointInTimeFinderOptions = { + type: 'alert', + perPage: maxPerPage, + namespaces: ['*'], + filter, + }; + logger.debug( + `Getting detection rules with point in time (PIT) query:', ${JSON.stringify(query)}` + ); + const finder = savedObjectsClient.createPointInTimeFinder(query); + let responses: Array> = []; + for await (const response of finder.find()) { + const extra = responses.length + response.saved_objects.length - maxSize; + if (extra > 0) { + responses = [ + ...responses, + ...response.saved_objects.slice(-response.saved_objects.length, -extra), + ]; + } else { + responses = [...responses, ...response.saved_objects]; + } + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + + logger.debug(`Returning cases response of length: "${responses.length}"`); + return responses; +}; diff --git a/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts new file mode 100644 index 0000000000000..6d720bef7d822 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/legacy_get_rule_actions.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsFindResult, + Logger, + SavedObjectsCreatePointInTimeFinderOptions, +} from 'kibana/server'; +// eslint-disable-next-line no-restricted-imports +import type { LegacyIRuleActionsAttributesSavedObjectAttributes } from '../../lib/detection_engine/rule_actions/legacy_types'; + +// eslint-disable-next-line no-restricted-imports +import { legacyRuleActionsSavedObjectType } from '../../lib/detection_engine/rule_actions/legacy_saved_object_mappings'; + +export interface LegacyGetRuleActionsOptions { + savedObjectsClient: SavedObjectsClientContract; + maxSize: number; + maxPerPage: number; + logger: Logger; +} + +/** + * Returns the legacy rule actions + * @deprecated Once we are confident all rules relying on side-car actions SO's have been migrated to SO references we should remove "legacyRuleActions" code including this function + */ +export const legacyGetRuleActions = async ({ + savedObjectsClient, + maxSize, + maxPerPage, + logger, +}: LegacyGetRuleActionsOptions) => { + const query: SavedObjectsCreatePointInTimeFinderOptions = { + type: legacyRuleActionsSavedObjectType, + perPage: maxPerPage, + namespaces: ['*'], + }; + logger.debug( + `Getting legacy rule actions with point in time (PIT) query:', ${JSON.stringify(query)}` + ); + const finder = + savedObjectsClient.createPointInTimeFinder( + query + ); + let responses: Array> = + []; + for await (const response of finder.find()) { + const extra = responses.length + response.saved_objects.length - maxSize; + if (extra > 0) { + responses = [ + ...responses, + ...response.saved_objects.slice(-response.saved_objects.length, -extra), + ]; + } else { + responses = [...responses, ...response.saved_objects]; + } + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + + logger.debug(`Returning legacy rule actions response of length: "${responses.length}"`); + return responses; +}; diff --git a/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts new file mode 100644 index 0000000000000..34dc545f9b8bf --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/utils/fetch_hits_with_pit.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + OpenPointInTimeResponse, + SearchHit, + SortResults, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; + +export interface FetchWithPitOptions { + esClient: ElasticsearchClient; + index: string; + maxSize: number; + maxPerPage: number; + searchRequest: SearchRequest; + logger: Logger; +} + +export const fetchHitsWithPit = async ({ + esClient, + index, + searchRequest, + maxSize, + maxPerPage, + logger, +}: FetchWithPitOptions): Promise>> => { + // default is from looking at Kibana saved objects and online documentation + const keepAlive = '5m'; + + // create and assign an initial point in time + let pitId: OpenPointInTimeResponse['id'] = ( + await esClient.openPointInTime({ + index, + keep_alive: '5m', + }) + ).id; + + let searchAfter: SortResults | undefined; + let hits: Array> = []; + let fetchMore = true; + while (fetchMore) { + const ruleSearchOptions: SearchRequest = { + ...searchRequest, + track_total_hits: false, + search_after: searchAfter, + sort: [{ _shard_doc: 'desc' }] as unknown as string[], // TODO: Remove this "unknown" once it is typed correctly https://github.com/elastic/elasticsearch-js/issues/1589 + pit: { id: pitId }, + size: Math.min(maxPerPage, maxSize - hits.length), + }; + logger.debug( + `Getting hits with point in time (PIT) query of: ${JSON.stringify(ruleSearchOptions)}` + ); + const body = await esClient.search(ruleSearchOptions); + hits = [...hits, ...body.hits.hits]; + searchAfter = + body.hits.hits.length !== 0 ? body.hits.hits[body.hits.hits.length - 1].sort : undefined; + + fetchMore = searchAfter != null && body.hits.hits.length > 0 && hits.length < maxSize; + if (body.pit_id != null) { + pitId = body.pit_id; + } + } + try { + await esClient.closePointInTime({ id: pitId }); + } catch (error) { + // Don't fail due to a bad point in time closure. We have seen failures in e2e tests during nominal operations. + logger.warn( + `Error trying to close point in time: "${pitId}", it will expire within "${keepAlive}". Error is: "${error}"` + ); + } + logger.debug(`Returning hits with point in time (PIT) length of: ${hits.length}`); + return hits; +}; diff --git a/x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts b/x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts new file mode 100644 index 0000000000000..f08959702b290 --- /dev/null +++ b/x-pack/plugins/security_solution/server/usage/queries/utils/is_elastic_rule.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; + +export const isElasticRule = (tags: string[] = []) => + tags.includes(`${INTERNAL_IMMUTABLE_KEY}:true`); diff --git a/x-pack/plugins/security_solution/server/usage/types.ts b/x-pack/plugins/security_solution/server/usage/types.ts index 1a3b5d1e2e29f..f591ffd8f422e 100644 --- a/x-pack/plugins/security_solution/server/usage/types.ts +++ b/x-pack/plugins/security_solution/server/usage/types.ts @@ -5,11 +5,49 @@ * 2.0. */ -import { CoreSetup } from 'src/core/server'; -import { SetupPlugins } from '../plugin'; +import type { CoreSetup, Logger } from 'src/core/server'; +import type { SanitizedAlert } from '../../../alerting/common/alert'; +import type { RuleParams } from '../lib/detection_engine/schemas/rule_schemas'; +import type { SetupPlugins } from '../plugin'; export type CollectorDependencies = { - kibanaIndex: string; signalsIndex: string; core: CoreSetup; + logger: Logger; } & Pick; + +export interface AlertBucket { + key: { + detectionAlerts: string; + }; + doc_count: number; +} + +export interface AlertAggs { + buckets?: { + after_key?: { + detectionAlerts: string; + }; + buckets: AlertBucket[]; + }; +} + +/** + * This type is _very_ similar to "RawRule". However, that type is not exposed in a non-restricted-path + * and it does not support generics well. Trying to use "RawRule" directly with TypeScript Omit does not work well. + * If at some point the rules client API supports cross spaces for gathering metrics, then we should remove our use + * of SavedObject types and this type below and instead natively use the rules client. + * + * NOTE: There are additional types not expressed below such as "apiKey" or there could be other slight differences + * but this will the easiest way to keep these in sync and I see other code that is similar to this pattern. + * {@see RawRule} + */ +export type RuleSearchResult = Omit< + SanitizedAlert, + 'createdBy' | 'updatedBy' | 'createdAt' | 'updatedAt' +> & { + createdBy: string | null; + updatedBy: string | null; + createdAt: string; + updatedAt: string; +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts index a8d473597a461..29baea4e4bd90 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/all_types.ts @@ -6,14 +6,14 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import type { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { createSignalsIndex, deleteAllAlerts, deleteSignalsIndex, getStats, } from '../../../../utils'; -import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; +import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/get_initial_usage'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts index 8a956d456edec..b93141a1ffe73 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/telemetry/usage_collector/detection_rules.ts @@ -6,12 +6,12 @@ */ import expect from '@kbn/expect'; -import { DetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/types'; -import { +import type { DetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/types'; +import type { ThreatMatchCreateSchema, ThresholdCreateSchema, } from '../../../../../../plugins/security_solution/common/detection_engine/schemas/request'; -import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import type { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { createLegacyRuleAction, createNewAction, @@ -33,7 +33,7 @@ import { waitForSignalsToBePresent, updateRule, } from '../../../../utils'; -import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/detection_rule_helpers'; +import { getInitialDetectionMetrics } from '../../../../../../plugins/security_solution/server/usage/detections/get_initial_usage'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => {