From 4f0a63f575f79d20f6ca32b2c2bb858dbd2526fc Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Thu, 2 Sep 2021 00:37:01 +0100 Subject: [PATCH] Security Telemetry Refactor (#109875) * [@pjhampton/@donaherc] Move sec telem tasks into own package. * Split filter out into its own module, started abstracting ES interaction into a queries module * Implemented querier and fixed some types * Updated tests, moved receiver to plugin from sender to decouple them. * fixed integration in detection engine, misc fixes * [@pjhampton] Fix type ref problems. Update test defs. * Make url transformer a member func of the sender class. * [@pjhampton] clean up receiver commentary. * [@pjhampton] add null check consistency. * Fix bad formatting. Co-authored-by: cdonaher Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../signals/send_telemetry_events.ts | 3 +- .../server/lib/telemetry/constants.ts | 2 + .../server/lib/telemetry/filters.ts | 124 ++++ .../server/lib/telemetry/mocks.ts | 31 +- .../server/lib/telemetry/receiver.ts | 276 +++++++++ .../server/lib/telemetry/sender.test.ts | 27 +- .../server/lib/telemetry/sender.ts | 542 ++---------------- .../diagnostic.test.ts} | 38 +- .../diagnostic.ts} | 15 +- .../endpoint.test.ts} | 40 +- .../{endpoint_task.ts => tasks/endpoint.ts} | 28 +- .../server/lib/telemetry/tasks/index.ts | 10 + .../security_lists.test.ts} | 36 +- .../security_lists.ts} | 20 +- .../server/lib/telemetry/types.ts | 53 ++ .../security_solution/server/plugin.ts | 11 +- 16 files changed, 686 insertions(+), 570 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/filters.ts create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts rename x-pack/plugins/security_solution/server/lib/telemetry/{diagnostic_task.test.ts => tasks/diagnostic.test.ts} (77%) rename x-pack/plugins/security_solution/server/lib/telemetry/{diagnostic_task.ts => tasks/diagnostic.ts} (87%) rename x-pack/plugins/security_solution/server/lib/telemetry/{endpoint_task.test.ts => tasks/endpoint.test.ts} (76%) rename x-pack/plugins/security_solution/server/lib/telemetry/{endpoint_task.ts => tasks/endpoint.ts} (93%) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts rename x-pack/plugins/security_solution/server/lib/telemetry/{security_lists_task.test.ts => tasks/security_lists.test.ts} (78%) rename x-pack/plugins/security_solution/server/lib/telemetry/{security_lists_task.ts => tasks/security_lists.ts} (87%) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts index d87427576cd8f..f9a5e4acb3160 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { TelemetryEventsSender, TelemetryEvent } from '../../telemetry/sender'; +import { TelemetryEventsSender } from '../../telemetry/sender'; +import { TelemetryEvent } from '../../telemetry/types'; import { BuildRuleMessage } from './rule_messages'; import { SignalSearchResponse, SignalSource } from './types'; import { Logger } from '../../../../../../../src/core/server'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts index 3ef45a554e7a5..1c03e52c67ae7 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/constants.ts @@ -5,6 +5,8 @@ * 2.0. */ +export const TELEMETRY_MAX_BUFFER_SIZE = 100; + export const TELEMETRY_CHANNEL_LISTS = 'security-lists'; export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts new file mode 100644 index 0000000000000..938772ce72367 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/filters.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface AllowlistFields { + [key: string]: boolean | AllowlistFields; +} + +// Allow list process fields within events. This includes "process" and "Target.process".' +const allowlistProcessFields: AllowlistFields = { + args: true, + name: true, + executable: true, + code_signature: true, + command_line: true, + hash: true, + pid: true, + pe: { + original_file_name: true, + }, + uptime: true, + Ext: { + architecture: true, + code_signature: true, + dll: true, + malware_signature: true, + memory_region: true, + token: { + integrity_level_name: true, + }, + }, + thread: true, + working_directory: true, +}; + +// Allow list for event-related fields, which can also be nested under events[] +const allowlistBaseEventFields: AllowlistFields = { + dll: { + name: true, + path: true, + code_signature: true, + malware_signature: true, + pe: { + original_file_name: true, + }, + }, + event: true, + file: { + extension: true, + name: true, + path: true, + size: true, + created: true, + accessed: true, + mtime: true, + directory: true, + hash: true, + Ext: { + code_signature: true, + header_data: true, + malware_classification: true, + malware_signature: true, + quarantine_result: true, + quarantine_message: true, + }, + }, + process: { + parent: allowlistProcessFields, + ...allowlistProcessFields, + }, + network: { + direction: true, + }, + registry: { + data: { + strings: true, + }, + hive: true, + key: true, + path: true, + value: true, + }, + Target: { + process: { + parent: allowlistProcessFields, + ...allowlistProcessFields, + }, + }, + user: { + id: true, + }, +}; + +// Allow list for the data we include in the events. True means that it is deep-cloned +// blindly. Object contents means that we only copy the fields that appear explicitly in +// the sub-object. +export const allowlistEventFields: AllowlistFields = { + '@timestamp': true, + agent: true, + Endpoint: true, + /* eslint-disable @typescript-eslint/naming-convention */ + Memory_protection: true, + Ransomware: true, + data_stream: true, + ecs: true, + elastic: true, + // behavioral protection re-nests some field sets under events.* (< 7.15) + events: allowlistBaseEventFields, + // behavioral protection re-nests some field sets under Events.* (>=7.15) + Events: allowlistBaseEventFields, + rule: { + id: true, + name: true, + ruleset: true, + version: true, + }, + host: { + os: true, + }, + ...allowlistBaseEventFields, +}; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts index a38042e214ceb..31903d5dafe93 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -7,9 +7,8 @@ // eslint-disable-next-line max-classes-per-file import { TelemetryEventsSender } from './sender'; -import { TelemetryDiagTask } from './diagnostic_task'; -import { TelemetryEndpointTask } from './endpoint_task'; -import { TelemetryExceptionListsTask } from './security_lists_task'; +import { TelemetryReceiver } from './receiver'; +import { DiagnosticTask, EndpointTask, ExceptionListsTask } from './tasks'; import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy'; /** @@ -22,20 +21,26 @@ export const createMockTelemetryEventsSender = ( setup: jest.fn(), start: jest.fn(), stop: jest.fn(), - fetchDiagnosticAlerts: jest.fn(), - fetchEndpointMetrics: jest.fn(), - fetchEndpointPolicyResponses: jest.fn(), - fetchTrustedApplications: jest.fn(), + fetchTelemetryUrl: jest.fn(), queueTelemetryEvents: jest.fn(), processEvents: jest.fn(), isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemtry ?? jest.fn()), sendIfDue: jest.fn(), + sendEvents: jest.fn(), + } as unknown) as jest.Mocked; +}; + +export const createMockTelemetryReceiver = (): jest.Mocked => { + return ({ + start: jest.fn(), fetchClusterInfo: jest.fn(), - fetchTelemetryUrl: jest.fn(), fetchLicenseInfo: jest.fn(), copyLicenseFields: jest.fn(), - sendEvents: jest.fn(), - } as unknown) as jest.Mocked; + fetchDiagnosticAlerts: jest.fn(), + fetchEndpointMetrics: jest.fn(), + fetchEndpointPolicyResponses: jest.fn(), + fetchTrustedApplications: jest.fn(), + } as unknown) as jest.Mocked; }; /** @@ -57,20 +62,20 @@ export const createMockPackagePolicy = (): jest.Mocked => { /** * Creates a mocked Telemetry Diagnostic Task */ -export class MockTelemetryDiagnosticTask extends TelemetryDiagTask { +export class MockTelemetryDiagnosticTask extends DiagnosticTask { public runTask = jest.fn(); } /** * Creates a mocked Telemetry Endpoint Task */ -export class MockTelemetryEndpointTask extends TelemetryEndpointTask { +export class MockTelemetryEndpointTask extends EndpointTask { public runTask = jest.fn(); } /** * Creates a mocked Telemetry exception lists Task */ -export class MockExceptionListsTask extends TelemetryExceptionListsTask { +export class MockExceptionListsTask extends ExceptionListsTask { public runTask = jest.fn(); } diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts new file mode 100644 index 0000000000000..dbdf8d6fe61ad --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts @@ -0,0 +1,276 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + Logger, + CoreStart, + ElasticsearchClient, + SavedObjectsClientContract, +} from 'src/core/server'; +import { SearchRequest } from '@elastic/elasticsearch/api/types'; +import { getTrustedAppsList } from '../../endpoint/routes/trusted_apps/service'; +import { AgentService, AgentPolicyServiceInterface } from '../../../../fleet/server'; +import { ExceptionListClient } from '../../../../lists/server'; +import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; +import { TELEMETRY_MAX_BUFFER_SIZE } from './constants'; +import { exceptionListItemToEndpointEntry } from './helpers'; +import { TelemetryEvent, ESLicense, ESClusterInfo, GetEndpointListResponse } from './types'; + +export class TelemetryReceiver { + private readonly logger: Logger; + private agentService?: AgentService; + private agentPolicyService?: AgentPolicyServiceInterface; + private esClient?: ElasticsearchClient; + private exceptionListClient?: ExceptionListClient; + private soClient?: SavedObjectsClientContract; + private readonly max_records = 10_000; + + constructor(logger: Logger) { + this.logger = logger.get('telemetry_events'); + } + + public async start( + core?: CoreStart, + endpointContextService?: EndpointAppContextService, + exceptionListClient?: ExceptionListClient + ) { + this.agentService = endpointContextService?.getAgentService(); + this.agentPolicyService = endpointContextService?.getAgentPolicyService(); + this.esClient = core?.elasticsearch.client.asInternalUser; + this.exceptionListClient = exceptionListClient; + this.soClient = (core?.savedObjects.createInternalRepository() as unknown) as SavedObjectsClientContract; + } + + public async fetchFleetAgents() { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve fleet policy responses'); + } + + return this.agentService?.listAgents(this.esClient, { + perPage: this.max_records, + showInactive: true, + sortField: 'enrolled_at', + sortOrder: 'desc', + }); + } + + public async fetchEndpointPolicyResponses(executeFrom: string, executeTo: string) { + if (this.esClient === undefined || this.esClient === null) { + throw Error( + 'elasticsearch client is unavailable: cannot retrieve elastic endpoint policy responses' + ); + } + + const query: SearchRequest = { + expand_wildcards: 'open,hidden', + index: `.ds-metrics-endpoint.policy*`, + ignore_unavailable: false, + size: 0, // no query results required - only aggregation quantity + body: { + query: { + range: { + '@timestamp': { + gte: executeFrom, + lt: executeTo, + }, + }, + }, + aggs: { + policy_responses: { + terms: { + size: this.max_records, + field: 'Endpoint.policy.applied.id', + }, + aggs: { + latest_response: { + top_hits: { + size: 1, + sort: [ + { + '@timestamp': { + order: 'desc' as const, + }, + }, + ], + }, + }, + }, + }, + }, + }, + }; + + return this.esClient.search(query); + } + + public async fetchEndpointMetrics(executeFrom: string, executeTo: string) { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve elastic endpoint metrics'); + } + + const query: SearchRequest = { + expand_wildcards: 'open,hidden', + index: `.ds-metrics-endpoint.metrics-*`, + ignore_unavailable: false, + size: 0, // no query results required - only aggregation quantity + body: { + query: { + range: { + '@timestamp': { + gte: executeFrom, + lt: executeTo, + }, + }, + }, + aggs: { + endpoint_agents: { + terms: { + field: 'agent.id', + size: this.max_records, + }, + aggs: { + latest_metrics: { + top_hits: { + size: 1, + sort: [ + { + '@timestamp': { + order: 'desc' as const, + }, + }, + ], + }, + }, + }, + }, + }, + }, + }; + + return this.esClient.search(query); + } + + public async fetchDiagnosticAlerts(executeFrom: string, executeTo: string) { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve diagnostic alerts'); + } + + const query = { + expand_wildcards: 'open,hidden', + index: '.logs-endpoint.diagnostic.collection-*', + ignore_unavailable: true, + size: TELEMETRY_MAX_BUFFER_SIZE, + body: { + query: { + range: { + 'event.ingested': { + gte: executeFrom, + lt: executeTo, + }, + }, + }, + sort: [ + { + 'event.ingested': { + order: 'desc' as const, + }, + }, + ], + }, + }; + + return (await this.esClient.search(query)).body; + } + + public async fetchPolicyConfigs(id: string) { + if (this.soClient === undefined || this.soClient === null) { + throw Error( + 'saved object client is unavailable: cannot retrieve endpoint policy configurations' + ); + } + + return this.agentPolicyService?.get(this.soClient, id); + } + + public async fetchTrustedApplications() { + if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { + throw Error('exception list client is unavailable: cannot retrieve trusted applications'); + } + + return getTrustedAppsList(this.exceptionListClient, { page: 1, per_page: 10_000 }); + } + + public async fetchEndpointList(listId: string): Promise { + if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { + throw Error('exception list client is unavailable: could not retrieve trusted applications'); + } + + // Ensure list is created if it does not exist + await this.exceptionListClient.createTrustedAppsList(); + + const results = await this.exceptionListClient.findExceptionListItem({ + listId, + page: 1, + perPage: this.max_records, + filter: undefined, + namespaceType: 'agnostic', + sortField: 'name', + sortOrder: 'asc', + }); + + return { + data: results?.data.map(exceptionListItemToEndpointEntry) ?? [], + total: results?.total ?? 0, + page: results?.page ?? 1, + per_page: results?.per_page ?? this.max_records, + }; + } + + public async fetchClusterInfo(): Promise { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation'); + } + + const { body } = await this.esClient.info(); + return body; + } + + public async fetchLicenseInfo(): Promise { + if (this.esClient === undefined || this.esClient === null) { + throw Error('elasticsearch client is unavailable: cannot retrieve license information'); + } + + try { + const ret = ( + await this.esClient.transport.request({ + method: 'GET', + path: '/_license', + querystring: { + local: true, + // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. + accept_enterprise: 'true', + }, + }) + ).body as Promise<{ license: ESLicense }>; + + return (await ret).license; + } catch (err) { + this.logger.debug(`failed retrieving license: ${err}`); + return undefined; + } + } + + public copyLicenseFields(lic: ESLicense) { + return { + uid: lic.uid, + status: lic.status, + type: lic.type, + ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), + ...(lic.issuer ? { issuer: lic.issuer } : {}), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts index 4e6520b67ab05..a4f8033f1160d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts @@ -6,7 +6,7 @@ */ /* eslint-disable dot-notation */ -import { TelemetryEventsSender, copyAllowlistedFields, getV3UrlFromV2 } from './sender'; +import { TelemetryEventsSender, copyAllowlistedFields } from './sender'; import { loggingSystemMock } from 'src/core/server/mocks'; import { usageCountersServiceMock } from 'src/plugins/usage_collection/server/usage_counters/usage_counters_service.mock'; import { URL } from 'url'; @@ -175,12 +175,6 @@ describe('TelemetryEventsSender', () => { getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')), }; sender['telemetryUsageCounter'] = telemetryUsageCounter; - sender['fetchClusterInfo'] = jest.fn(async () => { - return { - cluster_name: 'test', - cluster_uuid: 'test-uuid', - }; - }); sender['sendEvents'] = jest.fn(async () => { sender['telemetryUsageCounter']?.incrementCounter({ counterName: 'test_counter', @@ -339,21 +333,30 @@ describe('allowlistEventFields', () => { }); describe('getV3UrlFromV2', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + it('should return prod url', () => { - expect(getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint')).toBe( - 'https://telemetry.elastic.co/v3/send/alerts-endpoint' - ); + const sender = new TelemetryEventsSender(logger); + expect( + sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint') + ).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint'); }); it('should return staging url', () => { + const sender = new TelemetryEventsSender(logger); expect( - getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') + sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint') ).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint'); }); it('should support ports and auth', () => { + const sender = new TelemetryEventsSender(logger); expect( - getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint') + sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint') ).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint'); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index c7bb58dd2251b..2e615a2681174 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -7,10 +7,8 @@ import { cloneDeep } from 'lodash'; import axios from 'axios'; -import { SavedObjectsClientContract } from 'kibana/server'; -import { SearchRequest } from '@elastic/elasticsearch/api/types'; import { URL } from 'url'; -import { CoreStart, ElasticsearchClient, Logger } from 'src/core/server'; +import { Logger } from 'src/core/server'; import { TelemetryPluginStart, TelemetryPluginSetup } from 'src/plugins/telemetry/server'; import { UsageCounter } from 'src/plugins/usage_collection/server'; import { transformDataToNdjson } from '../../utils/read_stream/create_stream_from_ndjson'; @@ -18,65 +16,39 @@ import { TaskManagerSetupContract, TaskManagerStartContract, } from '../../../../task_manager/server'; -import { TelemetryDiagTask } from './diagnostic_task'; -import { TelemetryEndpointTask } from './endpoint_task'; -import { TelemetryExceptionListsTask } from './security_lists_task'; -import { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services'; -import { AgentService, AgentPolicyServiceInterface } from '../../../../fleet/server'; -import { getTrustedAppsList } from '../../endpoint/routes/trusted_apps/service'; -import { ExceptionListClient } from '../../../../lists/server'; -import { GetEndpointListResponse } from './types'; -import { createUsageCounterLabel, exceptionListItemToEndpointEntry } from './helpers'; - -type BaseSearchTypes = string | number | boolean | object; -export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined; -const usageLabelPrefix: string[] = ['security_telemetry', 'sender']; +import { TelemetryReceiver } from './receiver'; +import { AllowlistFields, allowlistEventFields } from './filters'; +import { DiagnosticTask, EndpointTask, ExceptionListsTask } from './tasks'; +import { createUsageCounterLabel } from './helpers'; +import { TelemetryEvent } from './types'; +import { TELEMETRY_MAX_BUFFER_SIZE } from './constants'; -export interface TelemetryEvent { - [key: string]: SearchTypes; - '@timestamp'?: string; - data_stream?: { - [key: string]: SearchTypes; - dataset?: string; - }; - cluster_name?: string; - cluster_uuid?: string; - file?: { - [key: string]: SearchTypes; - Ext?: { - [key: string]: SearchTypes; - }; - }; - license?: ESLicense; -} +const usageLabelPrefix: string[] = ['security_telemetry', 'sender']; export class TelemetryEventsSender { private readonly initialCheckDelayMs = 10 * 1000; private readonly checkIntervalMs = 60 * 1000; - private readonly max_records = 10_000; private readonly logger: Logger; - private maxQueueSize = 100; + private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE; private telemetryStart?: TelemetryPluginStart; private telemetrySetup?: TelemetryPluginSetup; private intervalId?: NodeJS.Timeout; private isSending = false; + private receiver: TelemetryReceiver | undefined; private queue: TelemetryEvent[] = []; private isOptedIn?: boolean = true; // Assume true until the first check - private diagTask?: TelemetryDiagTask; - private epMetricsTask?: TelemetryEndpointTask; - private exceptionListTask?: TelemetryExceptionListsTask; - private agentService?: AgentService; - private agentPolicyService?: AgentPolicyServiceInterface; - private esClient?: ElasticsearchClient; - private savedObjectsClient?: SavedObjectsClientContract; - private exceptionListClient?: ExceptionListClient; + private telemetryUsageCounter?: UsageCounter; + private diagnosticTask?: DiagnosticTask; + private endpointTask?: EndpointTask; + private exceptionListsTask?: ExceptionListsTask; constructor(logger: Logger) { this.logger = logger.get('telemetry_events'); } public setup( + telemetryReceiver: TelemetryReceiver, telemetrySetup?: TelemetryPluginSetup, taskManager?: TaskManagerSetupContract, telemetryUsageCounter?: UsageCounter @@ -85,31 +57,30 @@ export class TelemetryEventsSender { this.telemetryUsageCounter = telemetryUsageCounter; if (taskManager) { - this.diagTask = new TelemetryDiagTask(this.logger, taskManager, this); - this.epMetricsTask = new TelemetryEndpointTask(this.logger, taskManager, this); - this.exceptionListTask = new TelemetryExceptionListsTask(this.logger, taskManager, this); + this.diagnosticTask = new DiagnosticTask(this.logger, taskManager, this, telemetryReceiver); + this.endpointTask = new EndpointTask(this.logger, taskManager, this, telemetryReceiver); + this.exceptionListsTask = new ExceptionListsTask( + this.logger, + taskManager, + this, + telemetryReceiver + ); } } public start( - core?: CoreStart, telemetryStart?: TelemetryPluginStart, taskManager?: TaskManagerStartContract, - endpointContextService?: EndpointAppContextService, - exceptionListClient?: ExceptionListClient + receiver?: TelemetryReceiver ) { this.telemetryStart = telemetryStart; - this.esClient = core?.elasticsearch.client.asInternalUser; - this.agentService = endpointContextService?.getAgentService(); - this.agentPolicyService = endpointContextService?.getAgentPolicyService(); - this.savedObjectsClient = (core?.savedObjects.createInternalRepository() as unknown) as SavedObjectsClientContract; - this.exceptionListClient = exceptionListClient; - - if (taskManager && this.diagTask && this.epMetricsTask) { - this.logger.debug(`Starting diagnostic and endpoint telemetry tasks`); - this.diagTask.start(taskManager); - this.epMetricsTask.start(taskManager); - this.exceptionListTask?.start(taskManager); + this.receiver = receiver; + + if (taskManager && this.diagnosticTask && this.endpointTask && this.exceptionListsTask) { + this.logger.debug(`starting security telemetry tasks`); + this.diagnosticTask.start(taskManager); + this.endpointTask.start(taskManager); + this.exceptionListsTask?.start(taskManager); } this.logger.debug(`Starting local task`); @@ -125,187 +96,6 @@ export class TelemetryEventsSender { } } - public async fetchDiagnosticAlerts(executeFrom: string, executeTo: string) { - const query = { - expand_wildcards: 'open,hidden', - index: '.logs-endpoint.diagnostic.collection-*', - ignore_unavailable: true, - size: this.maxQueueSize, - body: { - query: { - range: { - 'event.ingested': { - gte: executeFrom, - lt: executeTo, - }, - }, - }, - sort: [ - { - 'event.ingested': { - order: 'desc' as const, - }, - }, - ], - }, - }; - - if (this.esClient === undefined) { - throw Error('could not fetch diagnostic alerts. es client is not available'); - } - - return (await this.esClient.search(query)).body; - } - - public async fetchEndpointMetrics(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('could not fetch policy responses. es client is not available'); - } - - const query: SearchRequest = { - expand_wildcards: 'open,hidden', - index: `.ds-metrics-endpoint.metrics-*`, - ignore_unavailable: false, - size: 0, // no query results required - only aggregation quantity - body: { - query: { - range: { - '@timestamp': { - gte: executeFrom, - lt: executeTo, - }, - }, - }, - aggs: { - endpoint_agents: { - terms: { - field: 'agent.id', - size: this.max_records, - }, - aggs: { - latest_metrics: { - top_hits: { - size: 1, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - }, - }, - }, - }, - }, - }; - - return this.esClient.search(query); - } - - public async fetchFleetAgents() { - if (this.esClient === undefined || this.esClient === null) { - throw Error('could not fetch policy responses. es client is not available'); - } - - return this.agentService?.listAgents(this.esClient, { - perPage: this.max_records, - showInactive: true, - sortField: 'enrolled_at', - sortOrder: 'desc', - }); - } - - public async fetchPolicyConfigs(id: string) { - if (this.savedObjectsClient === undefined || this.savedObjectsClient === null) { - throw Error('could not fetch endpoint policy configs. saved object client is not available'); - } - - return this.agentPolicyService?.get(this.savedObjectsClient, id); - } - - public async fetchEndpointPolicyResponses(executeFrom: string, executeTo: string) { - if (this.esClient === undefined || this.esClient === null) { - throw Error('could not fetch policy responses. es client is not available'); - } - - const query: SearchRequest = { - expand_wildcards: 'open,hidden', - index: `.ds-metrics-endpoint.policy*`, - ignore_unavailable: false, - size: 0, // no query results required - only aggregation quantity - body: { - query: { - range: { - '@timestamp': { - gte: executeFrom, - lt: executeTo, - }, - }, - }, - aggs: { - policy_responses: { - terms: { - size: this.max_records, - field: 'Endpoint.policy.applied.id', - }, - aggs: { - latest_response: { - top_hits: { - size: 1, - sort: [ - { - '@timestamp': { - order: 'desc', - }, - }, - ], - }, - }, - }, - }, - }, - }, - }; - - return this.esClient.search(query); - } - - public async fetchTrustedApplications() { - if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { - throw Error('could not fetch trusted applications. exception list client not available.'); - } - - return getTrustedAppsList(this.exceptionListClient, { page: 1, per_page: 10_000 }); - } - - public async fetchEndpointList(listId: string): Promise { - if (this?.exceptionListClient === undefined || this?.exceptionListClient === null) { - throw Error('could not fetch trusted applications. exception list client not available.'); - } - - // Ensure list is created if it does not exist - await this.exceptionListClient.createTrustedAppsList(); - - const results = await this.exceptionListClient.findExceptionListItem({ - listId, - page: 1, - perPage: this.max_records, - filter: undefined, - namespaceType: 'agnostic', - sortField: 'name', - sortOrder: 'asc', - }); - - return { - data: results?.data.map(exceptionListItemToEndpointEntry) ?? [], - total: results?.total ?? 0, - page: results?.page ?? 1, - per_page: results?.per_page ?? this.max_records, - }; - } - public queueTelemetryEvents(events: TelemetryEvent[]) { const qlength = this.queue.length; @@ -337,12 +127,6 @@ export class TelemetryEventsSender { } } - public processEvents(events: TelemetryEvent[]): TelemetryEvent[] { - return events.map(function (obj: TelemetryEvent): TelemetryEvent { - return copyAllowlistedFields(allowlistEventFields, obj); - }); - } - public async isTelemetryOptedIn() { this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); return this.isOptedIn === true; @@ -370,8 +154,8 @@ export class TelemetryEventsSender { const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ this.fetchTelemetryUrl('alerts-endpoint'), - this.fetchClusterInfo(), - this.fetchLicenseInfo(), + this.receiver?.fetchClusterInfo(), + this.receiver?.fetchLicenseInfo(), ]); this.logger.debug(`Telemetry URL: ${telemetryUrl}`); @@ -381,9 +165,9 @@ export class TelemetryEventsSender { const toSend: TelemetryEvent[] = cloneDeep(this.queue).map((event) => ({ ...event, - ...(licenseInfo ? { license: this.copyLicenseFields(licenseInfo) } : {}), - cluster_uuid: clusterInfo.cluster_uuid, - cluster_name: clusterInfo.cluster_name, + ...(licenseInfo ? { license: this.receiver?.copyLicenseFields(licenseInfo) } : {}), + cluster_uuid: clusterInfo?.cluster_uuid, + cluster_name: clusterInfo?.cluster_name, })); this.queue = []; @@ -391,8 +175,8 @@ export class TelemetryEventsSender { toSend, telemetryUrl, 'alerts-endpoint', - clusterInfo.cluster_uuid, - clusterInfo.version?.number, + clusterInfo?.cluster_uuid, + clusterInfo?.version?.number, licenseInfo?.uid ); } catch (err) { @@ -402,6 +186,12 @@ export class TelemetryEventsSender { this.isSending = false; } + public processEvents(events: TelemetryEvent[]): TelemetryEvent[] { + return events.map(function (obj: TelemetryEvent): TelemetryEvent { + return copyAllowlistedFields(allowlistEventFields, obj); + }); + } + /** * This function sends events to the elastic telemetry channel. Caution is required * because it does no allowlist filtering. The caller is responsible for making sure @@ -414,8 +204,8 @@ export class TelemetryEventsSender { try { const [telemetryUrl, clusterInfo, licenseInfo] = await Promise.all([ this.fetchTelemetryUrl(channel), - this.fetchClusterInfo(), - this.fetchLicenseInfo(), + this.receiver?.fetchClusterInfo(), + this.receiver?.fetchLicenseInfo(), ]); this.logger.debug(`Telemetry URL: ${telemetryUrl}`); @@ -427,8 +217,8 @@ export class TelemetryEventsSender { toSend, telemetryUrl, channel, - clusterInfo.cluster_uuid, - clusterInfo.version?.number, + clusterInfo?.cluster_uuid, + clusterInfo?.version?.number, licenseInfo?.uid ); } catch (err) { @@ -436,54 +226,36 @@ export class TelemetryEventsSender { } } - private async fetchClusterInfo(): Promise { - if (this.esClient === undefined) { - throw Error("Couldn't fetch cluster info. es client is not available"); - } - return getClusterInfo(this.esClient); - } - private async fetchTelemetryUrl(channel: string): Promise { const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl(); if (!telemetryUrl) { throw Error("Couldn't get telemetry URL"); } - return getV3UrlFromV2(telemetryUrl.toString(), channel); + return this.getV3UrlFromV2(telemetryUrl.toString(), channel); } - private async fetchLicenseInfo(): Promise { - if (!this.esClient) { - return undefined; - } - try { - const ret = await getLicense(this.esClient, true); - return ret.license; - } catch (err) { - this.logger.warn(`Error retrieving license: ${err}`); - return undefined; + // Forms URLs like: + // https://telemetry.elastic.co/v3/send/my-channel-name or + // https://telemetry-staging.elastic.co/v3-dev/send/my-channel-name + public getV3UrlFromV2(v2url: string, channel: string): string { + const url = new URL(v2url); + if (!url.hostname.includes('staging')) { + url.pathname = `/v3/send/${channel}`; + } else { + url.pathname = `/v3-dev/send/${channel}`; } - } - - private copyLicenseFields(lic: ESLicense) { - return { - uid: lic.uid, - status: lic.status, - type: lic.type, - ...(lic.issued_to ? { issued_to: lic.issued_to } : {}), - ...(lic.issuer ? { issuer: lic.issuer } : {}), - }; + return url.toString(); } private async sendEvents( events: unknown[], telemetryUrl: string, channel: string, - clusterUuid: string, + clusterUuid: string | undefined, clusterVersionNumber: string | undefined, licenseId: string | undefined ) { const ndjson = transformDataToNdjson(events); - // this.logger.debug(`NDJSON: ${ndjson}`); try { const resp = await axios.post(telemetryUrl, ndjson, { @@ -523,125 +295,6 @@ export class TelemetryEventsSender { } } -// For the Allowlist definition. -interface AllowlistFields { - [key: string]: boolean | AllowlistFields; -} - -// Allow list process fields within events. This includes "process" and "Target.process".' -const allowlistProcessFields: AllowlistFields = { - args: true, - name: true, - executable: true, - code_signature: true, - command_line: true, - hash: true, - pid: true, - pe: { - original_file_name: true, - }, - uptime: true, - Ext: { - architecture: true, - code_signature: true, - dll: true, - malware_signature: true, - memory_region: true, - token: { - integrity_level_name: true, - }, - }, - thread: true, - working_directory: true, -}; - -// Allow list for event-related fields, which can also be nested under events[] -const allowlistBaseEventFields: AllowlistFields = { - dll: { - name: true, - path: true, - code_signature: true, - malware_signature: true, - pe: { - original_file_name: true, - }, - }, - event: true, - file: { - extension: true, - name: true, - path: true, - size: true, - created: true, - accessed: true, - mtime: true, - directory: true, - hash: true, - Ext: { - code_signature: true, - header_data: true, - malware_classification: true, - malware_signature: true, - quarantine_result: true, - quarantine_message: true, - }, - }, - process: { - parent: allowlistProcessFields, - ...allowlistProcessFields, - }, - network: { - direction: true, - }, - registry: { - data: { - strings: true, - }, - hive: true, - key: true, - path: true, - value: true, - }, - Target: { - process: { - parent: allowlistProcessFields, - ...allowlistProcessFields, - }, - }, - user: { - id: true, - }, -}; - -// Allow list for the data we include in the events. True means that it is deep-cloned -// blindly. Object contents means that we only copy the fields that appear explicitly in -// the sub-object. -const allowlistEventFields: AllowlistFields = { - '@timestamp': true, - agent: true, - Endpoint: true, - /* eslint-disable @typescript-eslint/naming-convention */ - Memory_protection: true, - Ransomware: true, - data_stream: true, - ecs: true, - elastic: true, - // behavioral protection re-nests some field sets under events.* (< 7.15) - events: allowlistBaseEventFields, - // behavioral protection re-nests some field sets under Events.* (>=7.15) - Events: allowlistBaseEventFields, - rule: { - id: true, - name: true, - ruleset: true, - version: true, - }, - host: { - os: true, - }, - ...allowlistBaseEventFields, -}; - export function copyAllowlistedFields( allowlist: AllowlistFields, event: TelemetryEvent @@ -668,78 +321,3 @@ export function copyAllowlistedFields( return newEvent; }, {}); } - -// Forms URLs like: -// https://telemetry.elastic.co/v3/send/my-channel-name or -// https://telemetry-staging.elastic.co/v3-dev/send/my-channel-name -export function getV3UrlFromV2(v2url: string, channel: string): string { - const url = new URL(v2url); - if (!url.hostname.includes('staging')) { - url.pathname = `/v3/send/${channel}`; - } else { - url.pathname = `/v3-dev/send/${channel}`; - } - return url.toString(); -} - -// For getting cluster info. Copied from telemetry_collection/get_cluster_info.ts -export interface ESClusterInfo { - cluster_uuid: string; - cluster_name: string; - version?: { - number: string; - build_flavor: string; - build_type: string; - build_hash: string; - build_date: string; - build_snapshot?: boolean; - lucene_version: string; - minimum_wire_compatibility_version: string; - minimum_index_compatibility_version: string; - }; -} - -/** - * Get the cluster info from the connected cluster. - * Copied from: - * src/plugins/telemetry/server/telemetry_collection/get_cluster_info.ts - * This is the equivalent to GET / - * - * @param {function} esClient The asInternalUser handler (exposed for testing) - */ -export async function getClusterInfo(esClient: ElasticsearchClient) { - const { body } = await esClient.info(); - return body; -} - -// From https://www.elastic.co/guide/en/elasticsearch/reference/current/get-license.html -export interface ESLicense { - status: string; - uid: string; - type: string; - issue_date?: string; - issue_date_in_millis?: number; - expiry_date?: string; - expirty_date_in_millis?: number; - max_nodes?: number; - issued_to?: string; - issuer?: string; - start_date_in_millis?: number; -} - -async function getLicense( - esClient: ElasticsearchClient, - local: boolean -): Promise<{ license: ESLicense }> { - return ( - await esClient.transport.request({ - method: 'GET', - path: '/_license', - querystring: { - local, - // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license. - accept_enterprise: 'true', - }, - }) - ).body as Promise<{ license: ESLicense }>; // Note: We have to as cast since transport.request doesn't have generics -} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.test.ts similarity index 77% rename from x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.test.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.test.ts index 3008919fd2ced..fdf48bd99b3b2 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.test.ts @@ -6,10 +6,14 @@ */ import { loggingSystemMock } from 'src/core/server/mocks'; -import { taskManagerMock } from '../../../../task_manager/server/mocks'; -import { TaskStatus } from '../../../../task_manager/server'; -import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './diagnostic_task'; -import { createMockTelemetryEventsSender, MockTelemetryDiagnosticTask } from './mocks'; +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../../../task_manager/server'; +import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './diagnostic'; +import { + createMockTelemetryEventsSender, + MockTelemetryDiagnosticTask, + createMockTelemetryReceiver, +} from '../mocks'; describe('test', () => { let logger: ReturnType; @@ -23,7 +27,8 @@ describe('test', () => { const telemetryDiagTask = new TelemetryDiagTask( logger, taskManagerMock.createSetup(), - createMockTelemetryEventsSender(true) + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() ); expect(telemetryDiagTask).toBeInstanceOf(TelemetryDiagTask); @@ -32,7 +37,12 @@ describe('test', () => { test('diagnostic task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - new TelemetryDiagTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); + new TelemetryDiagTask( + logger, + mockTaskManager, + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() + ); expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); }); @@ -42,7 +52,8 @@ describe('test', () => { const telemetryDiagTask = new TelemetryDiagTask( logger, mockTaskManagerSetup, - createMockTelemetryEventsSender(true) + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() ); const mockTaskManagerStart = taskManagerMock.createStart(); @@ -53,7 +64,13 @@ describe('test', () => { test('diagnostic task should run', async () => { const mockContext = createMockTelemetryEventsSender(true); const mockTaskManager = taskManagerMock.createSetup(); - const telemetryDiagTask = new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockContext); + const mockReceiver = createMockTelemetryReceiver(); + const telemetryDiagTask = new MockTelemetryDiagnosticTask( + logger, + mockTaskManager, + mockContext, + mockReceiver + ); const mockTaskInstance = { id: TelemetryDiagTaskConstants.TYPE, @@ -79,7 +96,8 @@ describe('test', () => { test('diagnostic task should not query elastic if telemetry is not opted in', async () => { const mockSender = createMockTelemetryEventsSender(false); const mockTaskManager = taskManagerMock.createSetup(); - new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockSender); + const mockReceiver = createMockTelemetryReceiver(); + new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockSender, mockReceiver); const mockTaskInstance = { id: TelemetryDiagTaskConstants.TYPE, @@ -99,6 +117,6 @@ describe('test', () => { .createTaskRunner; const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); await taskRunner.run(); - expect(mockSender.fetchDiagnosticAlerts).not.toHaveBeenCalled(); + expect(mockReceiver.fetchDiagnosticAlerts).not.toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts similarity index 87% rename from x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts index f8c3ca914abe1..77d5ff72b04cb 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/diagnostic.ts @@ -11,9 +11,11 @@ import { ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, -} from '../../../../task_manager/server'; -import { getPreviousDiagTaskTimestamp } from './helpers'; -import { TelemetryEventsSender, TelemetryEvent } from './sender'; +} from '../../../../../task_manager/server'; +import { getPreviousDiagTaskTimestamp } from '../helpers'; +import { TelemetryEventsSender } from '../sender'; +import { TelemetryEvent } from '../types'; +import { TelemetryReceiver } from '../receiver'; export const TelemetryDiagTaskConstants = { TIMEOUT: '1m', @@ -25,14 +27,17 @@ export const TelemetryDiagTaskConstants = { export class TelemetryDiagTask { private readonly logger: Logger; private readonly sender: TelemetryEventsSender; + private readonly receiver: TelemetryReceiver; constructor( logger: Logger, taskManager: TaskManagerSetupContract, - sender: TelemetryEventsSender + sender: TelemetryEventsSender, + receiver: TelemetryReceiver ) { this.logger = logger; this.sender = sender; + this.receiver = receiver; taskManager.registerTaskDefinitions({ [TelemetryDiagTaskConstants.TYPE]: { @@ -99,7 +104,7 @@ export class TelemetryDiagTask { return 0; } - const response = await this.sender.fetchDiagnosticAlerts(searchFrom, searchTo); + const response = await this.receiver.fetchDiagnosticAlerts(searchFrom, searchTo); const hits = response.hits?.hits || []; if (!Array.isArray(hits) || !hits.length) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts similarity index 76% rename from x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts index 7366c94ce1c57..85f501d86f9c1 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.test.ts @@ -6,11 +6,15 @@ */ import { loggingSystemMock } from 'src/core/server/mocks'; -import { TaskStatus } from '../../../../task_manager/server'; -import { taskManagerMock } from '../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../../../task_manager/server'; +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; -import { TelemetryEndpointTask, TelemetryEndpointTaskConstants } from './endpoint_task'; -import { createMockTelemetryEventsSender, MockTelemetryEndpointTask } from './mocks'; +import { TelemetryEndpointTask, TelemetryEndpointTaskConstants } from './endpoint'; +import { + createMockTelemetryEventsSender, + MockTelemetryEndpointTask, + createMockTelemetryReceiver, +} from '../mocks'; describe('test', () => { let logger: ReturnType; @@ -24,7 +28,8 @@ describe('test', () => { const telemetryEndpointTask = new TelemetryEndpointTask( logger, taskManagerMock.createSetup(), - createMockTelemetryEventsSender(true) + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() ); expect(telemetryEndpointTask).toBeInstanceOf(TelemetryEndpointTask); @@ -33,7 +38,12 @@ describe('test', () => { test('the endpoint task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - new TelemetryEndpointTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); + new TelemetryEndpointTask( + logger, + mockTaskManager, + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() + ); expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); }); @@ -43,7 +53,8 @@ describe('test', () => { const telemetryEndpointTask = new TelemetryEndpointTask( logger, mockTaskManagerSetup, - createMockTelemetryEventsSender(true) + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() ); const mockTaskManagerStart = taskManagerMock.createStart(); @@ -54,7 +65,8 @@ describe('test', () => { test('endpoint task should not query elastic if telemetry is not opted in', async () => { const mockSender = createMockTelemetryEventsSender(false); const mockTaskManager = taskManagerMock.createSetup(); - new MockTelemetryEndpointTask(logger, mockTaskManager, mockSender); + const mockReceiver = createMockTelemetryReceiver(); + new MockTelemetryEndpointTask(logger, mockTaskManager, mockSender, mockReceiver); const mockTaskInstance = { id: TelemetryEndpointTaskConstants.TYPE, @@ -74,14 +86,20 @@ describe('test', () => { .createTaskRunner; const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); await taskRunner.run(); - expect(mockSender.fetchEndpointMetrics).not.toHaveBeenCalled(); - expect(mockSender.fetchEndpointPolicyResponses).not.toHaveBeenCalled(); + expect(mockReceiver.fetchEndpointMetrics).not.toHaveBeenCalled(); + expect(mockReceiver.fetchEndpointPolicyResponses).not.toHaveBeenCalled(); }); test('endpoint task should run when opted in', async () => { const mockSender = createMockTelemetryEventsSender(true); const mockTaskManager = taskManagerMock.createSetup(); - const telemetryEpMetaTask = new MockTelemetryEndpointTask(logger, mockTaskManager, mockSender); + const mockReceiver = createMockTelemetryReceiver(); + const telemetryEpMetaTask = new MockTelemetryEndpointTask( + logger, + mockTaskManager, + mockSender, + mockReceiver + ); const mockTaskInstance = { id: TelemetryEndpointTaskConstants.TYPE, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts similarity index 93% rename from x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts index 668696f0dce1d..67ca21b28a698 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/endpoint.ts @@ -11,21 +11,22 @@ import { ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, -} from '../../../../task_manager/server'; +} from '../../../../../task_manager/server'; import { batchTelemetryRecords, getPreviousEpMetaTaskTimestamp, isPackagePolicyList, -} from './helpers'; -import { TelemetryEventsSender } from './sender'; -import { PolicyData } from '../../../common/endpoint/types'; -import { FLEET_ENDPOINT_PACKAGE } from '../../../../fleet/common'; +} from '../helpers'; +import { TelemetryEventsSender } from '../sender'; +import { PolicyData } from '../../../../common/endpoint/types'; +import { FLEET_ENDPOINT_PACKAGE } from '../../../../../fleet/common'; import { EndpointMetricsAggregation, EndpointPolicyResponseAggregation, EndpointPolicyResponseDocument, -} from './types'; -import { TELEMETRY_CHANNEL_ENDPOINT_META } from './constants'; +} from '../types'; +import { TELEMETRY_CHANNEL_ENDPOINT_META } from '../constants'; +import { TelemetryReceiver } from '../receiver'; export const TelemetryEndpointTaskConstants = { TIMEOUT: '5m', @@ -53,14 +54,17 @@ const EmptyFleetAgentResponse = { export class TelemetryEndpointTask { private readonly logger: Logger; private readonly sender: TelemetryEventsSender; + private readonly receiver: TelemetryReceiver; constructor( logger: Logger, taskManager: TaskManagerSetupContract, - sender: TelemetryEventsSender + sender: TelemetryEventsSender, + receiver: TelemetryReceiver ) { this.logger = logger; this.sender = sender; + this.receiver = receiver; taskManager.registerTaskDefinitions({ [TelemetryEndpointTaskConstants.TYPE]: { @@ -121,9 +125,9 @@ export class TelemetryEndpointTask { private async fetchEndpointData(executeFrom: string, executeTo: string) { const [fleetAgentsResponse, epMetricsResponse, policyResponse] = await Promise.allSettled([ - this.sender.fetchFleetAgents(), - this.sender.fetchEndpointMetrics(executeFrom, executeTo), - this.sender.fetchEndpointPolicyResponses(executeFrom, executeTo), + this.receiver.fetchFleetAgents(), + this.receiver.fetchEndpointMetrics(executeFrom, executeTo), + this.receiver.fetchEndpointPolicyResponses(executeFrom, executeTo), ]); return { @@ -213,7 +217,7 @@ export class TelemetryEndpointTask { const endpointPolicyCache = new Map(); for (const policyInfo of fleetAgents.values()) { if (policyInfo !== null && policyInfo !== undefined && !endpointPolicyCache.has(policyInfo)) { - const agentPolicy = await this.sender.fetchPolicyConfigs(policyInfo); + const agentPolicy = await this.receiver.fetchPolicyConfigs(policyInfo); const packagePolicies = agentPolicy?.package_policies; if (packagePolicies !== undefined && isPackagePolicyList(packagePolicies)) { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts new file mode 100644 index 0000000000000..e090252b88d8f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { TelemetryDiagTask as DiagnosticTask } from './diagnostic'; +export { TelemetryEndpointTask as EndpointTask } from './endpoint'; +export { TelemetryExceptionListsTask as ExceptionListsTask } from './security_lists'; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.test.ts similarity index 78% rename from x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.test.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.test.ts index 20d89c9721b27..c54577cb8496e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.test.ts @@ -6,14 +6,14 @@ */ import { loggingSystemMock } from 'src/core/server/mocks'; -import { TaskStatus } from '../../../../task_manager/server'; -import { taskManagerMock } from '../../../../task_manager/server/mocks'; - +import { TaskStatus } from '../../../../../task_manager/server'; +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; +import { TelemetryExceptionListsTask, TelemetrySecuityListsTaskConstants } from './security_lists'; import { - TelemetryExceptionListsTask, - TelemetrySecuityListsTaskConstants, -} from './security_lists_task'; -import { createMockTelemetryEventsSender, MockExceptionListsTask } from './mocks'; + createMockTelemetryEventsSender, + MockExceptionListsTask, + createMockTelemetryReceiver, +} from '../mocks'; describe('test exception list telemetry task functionality', () => { let logger: ReturnType; @@ -26,7 +26,8 @@ describe('test exception list telemetry task functionality', () => { const telemetryTrustedAppsTask = new TelemetryExceptionListsTask( logger, taskManagerMock.createSetup(), - createMockTelemetryEventsSender(true) + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() ); expect(telemetryTrustedAppsTask).toBeInstanceOf(TelemetryExceptionListsTask); @@ -34,7 +35,12 @@ describe('test exception list telemetry task functionality', () => { test('the exception list task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - new TelemetryExceptionListsTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); + new TelemetryExceptionListsTask( + logger, + mockTaskManager, + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() + ); expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); }); @@ -44,7 +50,8 @@ describe('test exception list telemetry task functionality', () => { const telemetryTrustedAppsTask = new TelemetryExceptionListsTask( logger, mockTaskManagerSetup, - createMockTelemetryEventsSender(true) + createMockTelemetryEventsSender(true), + createMockTelemetryReceiver() ); const mockTaskManagerStart = taskManagerMock.createStart(); @@ -54,8 +61,9 @@ describe('test exception list telemetry task functionality', () => { test('the exception list task should not query elastic if telemetry is not opted in', async () => { const mockSender = createMockTelemetryEventsSender(false); + const mockReceiver = createMockTelemetryReceiver(); const mockTaskManager = taskManagerMock.createSetup(); - new MockExceptionListsTask(logger, mockTaskManager, mockSender); + new MockExceptionListsTask(logger, mockTaskManager, mockSender, mockReceiver); const mockTaskInstance = { id: TelemetrySecuityListsTaskConstants.TYPE, @@ -76,16 +84,18 @@ describe('test exception list telemetry task functionality', () => { ].createTaskRunner; const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); await taskRunner.run(); - expect(mockSender.fetchTrustedApplications).not.toHaveBeenCalled(); + expect(mockReceiver.fetchTrustedApplications).not.toHaveBeenCalled(); }); test('the exception list task should query elastic if telemetry opted in', async () => { const mockSender = createMockTelemetryEventsSender(true); const mockTaskManager = taskManagerMock.createSetup(); + const mockReceiver = createMockTelemetryReceiver(); const telemetryTrustedAppsTask = new MockExceptionListsTask( logger, mockTaskManager, - mockSender + mockSender, + mockReceiver ); const mockTaskInstance = { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts similarity index 87% rename from x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts rename to x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts index 1c4dc28f1c5a5..b54858e1f5f42 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/security_lists_task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts @@ -15,14 +15,15 @@ import { ConcreteTaskInstance, TaskManagerSetupContract, TaskManagerStartContract, -} from '../../../../task_manager/server'; +} from '../../../../../task_manager/server'; import { LIST_ENDPOINT_EXCEPTION, LIST_ENDPOINT_EVENT_FILTER, TELEMETRY_CHANNEL_LISTS, -} from './constants'; -import { batchTelemetryRecords, templateEndpointExceptions, templateTrustedApps } from './helpers'; -import { TelemetryEventsSender } from './sender'; +} from '../constants'; +import { batchTelemetryRecords, templateEndpointExceptions, templateTrustedApps } from '../helpers'; +import { TelemetryEventsSender } from '../sender'; +import { TelemetryReceiver } from '../receiver'; export const TelemetrySecuityListsTaskConstants = { TIMEOUT: '3m', @@ -36,14 +37,17 @@ const MAX_TELEMETRY_BATCH = 1_000; export class TelemetryExceptionListsTask { private readonly logger: Logger; private readonly sender: TelemetryEventsSender; + private readonly receiver: TelemetryReceiver; constructor( logger: Logger, taskManager: TaskManagerSetupContract, - sender: TelemetryEventsSender + sender: TelemetryEventsSender, + receiver: TelemetryReceiver ) { this.logger = logger; this.sender = sender; + this.receiver = receiver; taskManager.registerTaskDefinitions({ [TelemetrySecuityListsTaskConstants.TYPE]: { @@ -105,7 +109,7 @@ export class TelemetryExceptionListsTask { // Lists Telemetry: Trusted Applications - const trustedApps = await this.sender.fetchTrustedApplications(); + const trustedApps = await this.receiver.fetchTrustedApplications(); const trustedAppsJson = templateTrustedApps(trustedApps.data); this.logger.debug(`Trusted Apps: ${trustedAppsJson}`); @@ -115,7 +119,7 @@ export class TelemetryExceptionListsTask { // Lists Telemetry: Endpoint Exceptions - const epExceptions = await this.sender.fetchEndpointList(ENDPOINT_LIST_ID); + const epExceptions = await this.receiver.fetchEndpointList(ENDPOINT_LIST_ID); const epExceptionsJson = templateEndpointExceptions(epExceptions.data, LIST_ENDPOINT_EXCEPTION); this.logger.debug(`EP Exceptions: ${epExceptionsJson}`); @@ -125,7 +129,7 @@ export class TelemetryExceptionListsTask { // Lists Telemetry: Endpoint Event Filters - const epFilters = await this.sender.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID); + const epFilters = await this.receiver.fetchEndpointList(ENDPOINT_EVENT_FILTERS_LIST_ID); const epFiltersJson = templateEndpointExceptions(epFilters.data, LIST_ENDPOINT_EVENT_FILTER); this.logger.debug(`EP Event Filters: ${epFiltersJson}`); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts index d1d7740071e1f..b78017314a982 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts @@ -8,6 +8,59 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { TrustedApp } from '../../../common/endpoint/types'; +type BaseSearchTypes = string | number | boolean | object; +export type SearchTypes = BaseSearchTypes | BaseSearchTypes[] | undefined; + +// For getting cluster info. Copied from telemetry_collection/get_cluster_info.ts +export interface ESClusterInfo { + cluster_uuid: string; + cluster_name: string; + version?: { + number: string; + build_flavor: string; + build_type: string; + build_hash: string; + build_date: string; + build_snapshot?: boolean; + lucene_version: string; + minimum_wire_compatibility_version: string; + minimum_index_compatibility_version: string; + }; +} + +// From https://www.elastic.co/guide/en/elasticsearch/reference/current/get-license.html +export interface ESLicense { + status: string; + uid: string; + type: string; + issue_date?: string; + issue_date_in_millis?: number; + expiry_date?: string; + expirty_date_in_millis?: number; + max_nodes?: number; + issued_to?: string; + issuer?: string; + start_date_in_millis?: number; +} + +export interface TelemetryEvent { + [key: string]: SearchTypes; + '@timestamp'?: string; + data_stream?: { + [key: string]: SearchTypes; + dataset?: string; + }; + cluster_name?: string; + cluster_uuid?: string; + file?: { + [key: string]: SearchTypes; + Ext?: { + [key: string]: SearchTypes; + }; + }; + license?: ESLicense; +} + // EP Policy Response export interface EndpointPolicyResponseAggregation { diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 040ebb659abce..d657d7e06b1a6 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -81,6 +81,7 @@ import type { SecuritySolutionRequestHandlerContext } from './types'; import { registerTrustedAppsRoutes } from './endpoint/routes/trusted_apps'; import { securitySolutionSearchStrategyProvider } from './search_strategy/security_solution'; import { TelemetryEventsSender } from './lib/telemetry/sender'; +import { TelemetryReceiver } from './lib/telemetry/receiver'; import { TelemetryPluginStart, TelemetryPluginSetup, @@ -139,6 +140,7 @@ export class Plugin implements IPlugin({ max: 3, maxAge: 1000 * 60 * 5 }); this.telemetryEventsSender = new TelemetryEventsSender(this.logger); + this.telemetryReceiver = new TelemetryReceiver(this.logger); this.logger.debug('plugin initialized'); } @@ -328,6 +331,7 @@ export class Plugin implements IPlugin