Security Telemetry Refactor (#109875)

* [@pjhampton/@donaherc] Move sec telem tasks into own package.

* Split filter out into its own module, started abstracting ES interaction into a queries module

* Implemented querier and fixed some types

* Updated tests, moved receiver to plugin from sender to decouple them.

* fixed integration in detection engine, misc fixes

* [@pjhampton] Fix type ref problems. Update test defs.

* Make url transformer a member func of the sender class.

* [@pjhampton] clean up receiver commentary.

* [@pjhampton] add null check consistency.

* Fix bad formatting.

Co-authored-by: cdonaher <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts

@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-import { TelemetryEventsSender, TelemetryEvent } from '../../telemetry/sender';
+import { TelemetryEventsSender } from '../../telemetry/sender';
+import { TelemetryEvent } from '../../telemetry/types';
 import { BuildRuleMessage } from './rule_messages';
 import { SignalSearchResponse, SignalSource } from './types';
 import { Logger } from '../../../../../../../src/core/server';

x-pack/plugins/security_solution/server/lib/telemetry/constants.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+export const TELEMETRY_MAX_BUFFER_SIZE = 100;
+
 export const TELEMETRY_CHANNEL_LISTS = 'security-lists';
 
 export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata';

x-pack/plugins/security_solution/server/lib/telemetry/filters.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface AllowlistFields {
+  [key: string]: boolean | AllowlistFields;
+}
+
+// Allow list process fields within events.  This includes "process" and "Target.process".'
+const allowlistProcessFields: AllowlistFields = {
+  args: true,
+  name: true,
+  executable: true,
+  code_signature: true,
+  command_line: true,
+  hash: true,
+  pid: true,
+  pe: {
+    original_file_name: true,
+  },
+  uptime: true,
+  Ext: {
+    architecture: true,
+    code_signature: true,
+    dll: true,
+    malware_signature: true,
+    memory_region: true,
+    token: {
+      integrity_level_name: true,
+    },
+  },
+  thread: true,
+  working_directory: true,
+};
+
+// Allow list for event-related fields, which can also be nested under events[]
+const allowlistBaseEventFields: AllowlistFields = {
+  dll: {
+    name: true,
+    path: true,
+    code_signature: true,
+    malware_signature: true,
+    pe: {
+      original_file_name: true,
+    },
+  },
+  event: true,
+  file: {
+    extension: true,
+    name: true,
+    path: true,
+    size: true,
+    created: true,
+    accessed: true,
+    mtime: true,
+    directory: true,
+    hash: true,
+    Ext: {
+      code_signature: true,
+      header_data: true,
+      malware_classification: true,
+      malware_signature: true,
+      quarantine_result: true,
+      quarantine_message: true,
+    },
+  },
+  process: {
+    parent: allowlistProcessFields,
+    ...allowlistProcessFields,
+  },
+  network: {
+    direction: true,
+  },
+  registry: {
+    data: {
+      strings: true,
+    },
+    hive: true,
+    key: true,
+    path: true,
+    value: true,
+  },
+  Target: {
+    process: {
+      parent: allowlistProcessFields,
+      ...allowlistProcessFields,
+    },
+  },
+  user: {
+    id: true,
+  },
+};
+
+// Allow list for the data we include in the events. True means that it is deep-cloned
+// blindly. Object contents means that we only copy the fields that appear explicitly in
+// the sub-object.
+export const allowlistEventFields: AllowlistFields = {
+  '@timestamp': true,
+  agent: true,
+  Endpoint: true,
+  /* eslint-disable @typescript-eslint/naming-convention */
+  Memory_protection: true,
+  Ransomware: true,
+  data_stream: true,
+  ecs: true,
+  elastic: true,
+  // behavioral protection re-nests some field sets under events.* (< 7.15)
+  events: allowlistBaseEventFields,
+  // behavioral protection re-nests some field sets under Events.* (>=7.15)
+  Events: allowlistBaseEventFields,
+  rule: {
+    id: true,
+    name: true,
+    ruleset: true,
+    version: true,
+  },
+  host: {
+    os: true,
+  },
+  ...allowlistBaseEventFields,
+};

x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts

@@ -7,9 +7,8 @@
 
 // eslint-disable-next-line max-classes-per-file
 import { TelemetryEventsSender } from './sender';
-import { TelemetryDiagTask } from './diagnostic_task';
-import { TelemetryEndpointTask } from './endpoint_task';
-import { TelemetryExceptionListsTask } from './security_lists_task';
+import { TelemetryReceiver } from './receiver';
+import { DiagnosticTask, EndpointTask, ExceptionListsTask } from './tasks';
 import { PackagePolicy } from '../../../../fleet/common/types/models/package_policy';
 
 /**
@@ -22,20 +21,26 @@ export const createMockTelemetryEventsSender = (
     setup: jest.fn(),
     start: jest.fn(),
     stop: jest.fn(),
-    fetchDiagnosticAlerts: jest.fn(),
-    fetchEndpointMetrics: jest.fn(),
-    fetchEndpointPolicyResponses: jest.fn(),
-    fetchTrustedApplications: jest.fn(),
+    fetchTelemetryUrl: jest.fn(),
     queueTelemetryEvents: jest.fn(),
     processEvents: jest.fn(),
     isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemtry ?? jest.fn()),
     sendIfDue: jest.fn(),
+    sendEvents: jest.fn(),
+  } as unknown) as jest.Mocked<TelemetryEventsSender>;
+};
+
+export const createMockTelemetryReceiver = (): jest.Mocked<TelemetryReceiver> => {
+  return ({
+    start: jest.fn(),
     fetchClusterInfo: jest.fn(),
-    fetchTelemetryUrl: jest.fn(),
     fetchLicenseInfo: jest.fn(),
     copyLicenseFields: jest.fn(),
-    sendEvents: jest.fn(),
-  } as unknown) as jest.Mocked<TelemetryEventsSender>;
+    fetchDiagnosticAlerts: jest.fn(),
+    fetchEndpointMetrics: jest.fn(),
+    fetchEndpointPolicyResponses: jest.fn(),
+    fetchTrustedApplications: jest.fn(),
+  } as unknown) as jest.Mocked<TelemetryReceiver>;
 };
 
 /**
@@ -57,20 +62,20 @@ export const createMockPackagePolicy = (): jest.Mocked<PackagePolicy> => {
 /**
  * Creates a mocked Telemetry Diagnostic Task
  */
-export class MockTelemetryDiagnosticTask extends TelemetryDiagTask {
+export class MockTelemetryDiagnosticTask extends DiagnosticTask {
   public runTask = jest.fn();
 }
 
 /**
  * Creates a mocked Telemetry Endpoint Task
  */
-export class MockTelemetryEndpointTask extends TelemetryEndpointTask {
+export class MockTelemetryEndpointTask extends EndpointTask {
   public runTask = jest.fn();
 }
 
 /**
  * Creates a mocked Telemetry exception lists Task
  */
-export class MockExceptionListsTask extends TelemetryExceptionListsTask {
+export class MockExceptionListsTask extends ExceptionListsTask {
   public runTask = jest.fn();
 }