[SIEM][Security Solution][Endpoint] Endpoint Artifact Manifest Manage…

…ment + Artifact Download and Distribution (#67707) (#70758)

* stub out task for the exceptions list packager

* Hits list code and pages

* refactor

* Begin adding saved object and type definitions

* Transforms to endpoint exceptions

* Get internal SO client

* update messaging

* cleanup

* Integrating with task manager

* Integrated with task manager properly

* Begin adding schemas

* Add multiple OS and schema version support

* filter by OS

* Fixing sort

* Move to security_solutions

* siem -> securitySolution

* Progress on downloads, cleanup

* Add config, update artifact creation, add TODOs

* Fixing buffer serialization problem

* Adding cleanup to task

* Handle HEAD req

* proper header

* More robust task management

* single -> agnostic

* Fix OS filtering

* Scaffolding digital signatures / tests

* Adds rotue for creating endpoint user

* Cleanup

* persisting user

* Adding route to fetch created user

* Addings tests for translating exceptions

* Adding test for download API

* Download tweaks + artifact generation fixes

* reorganize

* fix imports

* Fixing test

* Changes id of SO

* integration tests setup

* Add first integration tests

* Cache layer

* more schema validation

* Set up for manifest update

* minor change

* remove setup code

* add manifest schema

* refactoring

* manifest rewrite (partial)

* finish scaffolding new manifest logic

* syntax errors

* more refactoring

* Move to endpoint directory

* minor cleanup

* clean up old artifacts

* Use diff appropriately

* Fix download

* schedule task on interval

* Split up into client/manager

* more mocks

* config interval

* Fixing download tests and adding cache tests

* lint

* mo money, mo progress

* Converting to io-ts

* More tests and mocks

* even more tests and mocks

* Merging both refactors

* Adding more tests for the convertion layer

* fix conflicts

* Adding lzma types

* Bug fixes

* lint

* resolve some type errors

* Adding back in cache

* Fixing download test

* Changing cache to be sized

* Fix manifest manager initialization

* Hook up datasource service

* Fix download tests

* Incremental progress

* Adds integration with ingest manager for auth

* Update test fixture

* Add manifest dispatch

* Refactoring to use the same SO Client from ingest

* bug fixes

* build renovate config

* Fix endpoint_app_context_services tests

* Only index the fields that are necessary for searching

* Integ test progress

* mock and test city

* Add task tests

* Tests for artifact_client and manifest_client

* Add manifest_manager tests

* minor refactor

* Finish manifest_manager tests

* Type errors

* Update integ test

* Type errors, final cleanup

* Fix integration test and add test for invalid api key

* minor fixup

* Remove compression

* Update task interval

* Removing .text suffix from translated list

* Fixes hashes for unit tests

* clean up yarn.lock

* Remove lzma-native from package.json

* missed updating one of the tests

Co-authored-by: Alex Kahan <[email protected]>

Co-authored-by: Alex Kahan <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/lists/server/mocks.ts

@@ -18,6 +18,6 @@ const createSetupMock = (): jest.Mocked<ListPluginSetup> => {
 
 export const listMock = {
   createSetup: createSetupMock,
-  getExceptionList: getExceptionListClientMock,
+  getExceptionListClient: getExceptionListClientMock,
   getListClient: getListClientMock,
 };

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -1034,6 +1034,13 @@ export class EndpointDocGenerator {
           enabled: true,
           streams: [],
           config: {
+            artifact_manifest: {
+              value: {
+                manifest_version: 'v0',
+                schema_version: '1.0.0',
+                artifacts: {},
+              },
+            },
             policy: {
               value: policyFactory(),
             },

x-pack/plugins/security_solution/common/endpoint/schema/common.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+export const identifier = t.string;
+
+export const manifestVersion = t.string;
+
+export const manifestSchemaVersion = t.keyof({
+  '1.0.0': null,
+});
+export type ManifestSchemaVersion = t.TypeOf<typeof manifestSchemaVersion>;
+
+export const sha256 = t.string;
+
+export const size = t.number;
+
+export const url = t.string;

x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common';
+
+export const manifestEntrySchema = t.exact(
+  t.type({
+    url,
+    sha256,
+    size,
+  })
+);
+
+export const manifestSchema = t.exact(
+  t.type({
+    manifest_version: manifestVersion,
+    schema_version: manifestSchemaVersion,
+    artifacts: t.record(identifier, manifestEntrySchema),
+  })
+);
+
+export type ManifestEntrySchema = t.TypeOf<typeof manifestEntrySchema>;
+export type ManifestSchema = t.TypeOf<typeof manifestSchema>;

x-pack/plugins/security_solution/common/endpoint/types.ts

@@ -5,6 +5,7 @@
  */
 
 import { PackageConfig, NewPackageConfig } from '../../../ingest_manager/common';
+import { ManifestSchema } from './schema/manifest';
 
 /**
  * Object that allows you to maintain stateful information in the location object across navigation events
@@ -683,6 +684,9 @@ export type NewPolicyData = NewPackageConfig & {
       enabled: boolean;
       streams: [];
       config: {
+        artifact_manifest: {
+          value: ManifestSchema;
+        };
         policy: {
           value: PolicyConfig;
         };

x-pack/plugins/security_solution/kibana.json

@@ -12,6 +12,7 @@
     "features",
     "home",
     "ingestManager",
+    "taskManager",
     "inspector",
     "licensing",
     "maps",

x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts

@@ -41,6 +41,13 @@ describe('policy details: ', () => {
               enabled: true,
               streams: [],
               config: {
+                artifact_manifest: {
+                  value: {
+                    manifest_version: 'v0',
+                    schema_version: '1.0.0',
+                    artifacts: {},
+                  },
+                },
                 policy: {
                   value: policyConfigFactory(),
                 },

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts

@@ -3,11 +3,23 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
+import { httpServerMock } from '../../../../../src/core/server/mocks';
 import { EndpointAppContextService } from './endpoint_app_context_services';
 
 describe('test endpoint app context services', () => {
-  it('should throw error if start is not called', async () => {
+  it('should throw error on getAgentService if start is not called', async () => {
     const endpointAppContextService = new EndpointAppContextService();
     expect(() => endpointAppContextService.getAgentService()).toThrow(Error);
   });
+  it('should return undefined on getManifestManager if start is not called', async () => {
+    const endpointAppContextService = new EndpointAppContextService();
+    expect(endpointAppContextService.getManifestManager()).toEqual(undefined);
+  });
+  it('should throw error on getScopedSavedObjectsClient if start is not called', async () => {
+    const endpointAppContextService = new EndpointAppContextService();
+    expect(() =>
+      endpointAppContextService.getScopedSavedObjectsClient(httpServerMock.createKibanaRequest())
+    ).toThrow(Error);
+  });
 });

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -3,14 +3,22 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import {
+  SavedObjectsServiceStart,
+  KibanaRequest,
+  SavedObjectsClientContract,
+} from 'src/core/server';
 import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
-import { handlePackageConfigCreate } from './ingest_integration';
+import { getPackageConfigCreateCallback } from './ingest_integration';
+import { ManifestManager } from './services/artifacts';
 
 export type EndpointAppContextServiceStartContract = Pick<
   IngestManagerStartContract,
   'agentService'
 > & {
+  manifestManager?: ManifestManager | undefined;
   registerIngestCallback: IngestManagerStartContract['registerExternalCallback'];
+  savedObjectsStart: SavedObjectsServiceStart;
 };
 
 /**
@@ -19,10 +27,20 @@ export type EndpointAppContextServiceStartContract = Pick<
  */
 export class EndpointAppContextService {
   private agentService: AgentService | undefined;
+  private manifestManager: ManifestManager | undefined;
+  private savedObjectsStart: SavedObjectsServiceStart | undefined;
 
   public start(dependencies: EndpointAppContextServiceStartContract) {
     this.agentService = dependencies.agentService;
-    dependencies.registerIngestCallback('packageConfigCreate', handlePackageConfigCreate);
+    this.manifestManager = dependencies.manifestManager;
+    this.savedObjectsStart = dependencies.savedObjectsStart;
+
+    if (this.manifestManager !== undefined) {
+      dependencies.registerIngestCallback(
+        'packageConfigCreate',
+        getPackageConfigCreateCallback(this.manifestManager)
+      );
+    }
   }
 
   public stop() {}
@@ -33,4 +51,15 @@ export class EndpointAppContextService {
     }
     return this.agentService;
   }
+
+  public getManifestManager(): ManifestManager | undefined {
+    return this.manifestManager;
+  }
+
+  public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClientContract {
+    if (!this.savedObjectsStart) {
+      throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`);
+    }
+    return this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] });
+  }
 }

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -4,46 +4,62 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { NewPackageConfig } from '../../../ingest_manager/common/types/models';
 import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
 import { NewPolicyData } from '../../common/endpoint/types';
-import { NewPackageConfig } from '../../../ingest_manager/common/types/models';
+import { ManifestManager } from './services/artifacts';
 
 /**
- * Callback to handle creation of package configs in Ingest Manager
- * @param newPackageConfig
+ * Callback to handle creation of PackageConfigs in Ingest Manager
  */
-export const handlePackageConfigCreate = async (
-  newPackageConfig: NewPackageConfig
-): Promise<NewPackageConfig> => {
-  // We only care about Endpoint package configs
-  if (newPackageConfig.package?.name !== 'endpoint') {
-    return newPackageConfig;
-  }
+export const getPackageConfigCreateCallback = (
+  manifestManager: ManifestManager
+): ((newPackageConfig: NewPackageConfig) => Promise<NewPackageConfig>) => {
+  const handlePackageConfigCreate = async (
+    newPackageConfig: NewPackageConfig
+  ): Promise<NewPackageConfig> => {
+    // We only care about Endpoint package configs
+    if (newPackageConfig.package?.name !== 'endpoint') {
+      return newPackageConfig;
+    }
 
-  // We cast the type here so that any changes to the Endpoint specific data
-  // follow the types/schema expected
-  let updatedPackageConfig = newPackageConfig as NewPolicyData;
+    // We cast the type here so that any changes to the Endpoint specific data
+    // follow the types/schema expected
+    let updatedPackageConfig = newPackageConfig as NewPolicyData;
 
-  // Until we get the Default Policy Configuration in the Endpoint package,
-  // we will add it here manually at creation time.
-  // @ts-ignore
-  if (newPackageConfig.inputs.length === 0) {
-    updatedPackageConfig = {
-      ...newPackageConfig,
-      inputs: [
-        {
-          type: 'endpoint',
-          enabled: true,
-          streams: [],
-          config: {
-            policy: {
-              value: policyConfigFactory(),
+    const wrappedManifest = await manifestManager.refresh({ initialize: true });
+    if (wrappedManifest !== null) {
+      // Until we get the Default Policy Configuration in the Endpoint package,
+      // we will add it here manually at creation time.
+      // @ts-ignore
+      if (newPackageConfig.inputs.length === 0) {
+        updatedPackageConfig = {
+          ...newPackageConfig,
+          inputs: [
+            {
+              type: 'endpoint',
+              enabled: true,
+              streams: [],
+              config: {
+                artifact_manifest: {
+                  value: wrappedManifest.manifest.toEndpointFormat(),
+                },
+                policy: {
+                  value: policyConfigFactory(),
+                },
+              },
             },
-          },
-        },
-      ],
-    };
-  }
+          ],
+        };
+      }
+    }
+
+    try {
+      return updatedPackageConfig;
+    } finally {
+      await manifestManager.commit(wrappedManifest);
+    }
+  };
 
-  return updatedPackageConfig;
+  return handlePackageConfigCreate;
 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ExceptionsCache } from './cache';
+
+describe('ExceptionsCache tests', () => {
+  let cache: ExceptionsCache;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    cache = new ExceptionsCache(3);
+  });
+
+  test('it should cache', async () => {
+    cache.set('test', 'body');
+    const cacheResp = cache.get('test');
+    expect(cacheResp).toEqual('body');
+  });
+
+  test('it should handle cache miss', async () => {
+    cache.set('test', 'body');
+    const cacheResp = cache.get('not test');
+    expect(cacheResp).toEqual(undefined);
+  });
+
+  test('it should handle cache eviction', async () => {
+    cache.set('1', 'a');
+    cache.set('2', 'b');
+    cache.set('3', 'c');
+    const cacheResp = cache.get('1');
+    expect(cacheResp).toEqual('a');
+
+    cache.set('4', 'd');
+    const secondResp = cache.get('1');
+    expect(secondResp).toEqual(undefined);
+    expect(cache.get('2')).toEqual('b');
+    expect(cache.get('3')).toEqual('c');
+    expect(cache.get('4')).toEqual('d');
+  });
+});

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+const DEFAULT_MAX_SIZE = 10;
+
+/**
+ * FIFO cache implementation for artifact downloads.
+ */
+export class ExceptionsCache {
+  private cache: Map<string, string>;
+  private queue: string[];
+  private maxSize: number;
+
+  constructor(maxSize: number) {
+    this.cache = new Map();
+    this.queue = [];
+    this.maxSize = maxSize || DEFAULT_MAX_SIZE;
+  }
+
+  set(id: string, body: string) {
+    if (this.queue.length + 1 > this.maxSize) {
+      const entry = this.queue.shift();
+      if (entry !== undefined) {
+        this.cache.delete(entry);
+      }
+    }
+    this.queue.push(id);
+    this.cache.set(id, body);
+  }
+
+  get(id: string): string | undefined {
+    return this.cache.get(id);
+  }
+}

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const ArtifactConstants = {
+  GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
+  SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact',
+  SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
+  SCHEMA_VERSION: '1.0.0',
+};
+
+export const ManifestConstants = {
+  SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest',
+  SCHEMA_VERSION: '1.0.0',
+};