[SIEM][Security Solution][Endpoint] Endpoint Artifact Manifest Management + Artifact Download and Distribution

x-pack/plugins/lists/server/mocks.ts

@@ -18,6 +18,6 @@ const createSetupMock = (): jest.Mocked<ListPluginSetup> => {
 
 export const listMock = {
   createSetup: createSetupMock,
-  getExceptionList: getExceptionListClientMock,
+  getExceptionListClient: getExceptionListClientMock,
   getListClient: getListClientMock,
 };

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -986,6 +986,13 @@ export class EndpointDocGenerator {
           enabled: true,
           streams: [],
           config: {
+            artifact_manifest: {
+              value: {
+                manifest_version: 'v0',
+                schema_version: '1.0.0',
+                artifacts: {},
+              },
+            },
             policy: {
               value: policyFactory(),
             },

x-pack/plugins/security_solution/common/endpoint/schema/common.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+export const identifier = t.string;
+
+export const manifestVersion = t.string;
+
+export const manifestSchemaVersion = t.keyof({
+  '1.0.0': null,
+});
+export type ManifestSchemaVersion = t.TypeOf<typeof manifestSchemaVersion>;
+
+export const sha256 = t.string;
+
+export const size = t.number;
+
+export const url = t.string;

x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common';
+
+export const manifestEntrySchema = t.exact(
+  t.type({
+    url,
+    sha256,
+    size,
+  })
+);
+
+export const manifestSchema = t.exact(
+  t.type({
+    manifest_version: manifestVersion,
+    schema_version: manifestSchemaVersion,
+    artifacts: t.record(identifier, manifestEntrySchema),
+  })
+);
+
+export type ManifestEntrySchema = t.TypeOf<typeof manifestEntrySchema>;
+export type ManifestSchema = t.TypeOf<typeof manifestSchema>;

x-pack/plugins/security_solution/common/endpoint/types.ts

@@ -5,6 +5,7 @@
  */
 
 import { Datasource, NewDatasource } from '../../../ingest_manager/common';
+import { ManifestSchema } from './schema/manifest';
 
 /**
  * Object that allows you to maintain stateful information in the location object across navigation events
@@ -684,6 +685,9 @@ export type NewPolicyData = NewDatasource & {
       enabled: boolean;
       streams: [];
       config: {
+        artifact_manifest: {
+          value: ManifestSchema;
+        };
         policy: {
           value: PolicyConfig;
         };

x-pack/plugins/security_solution/kibana.json

@@ -12,6 +12,7 @@
     "features",
     "home",
     "ingestManager",
+    "taskManager",
     "inspector",
     "licensing",
     "maps",

x-pack/plugins/security_solution/package.json

@@ -20,8 +20,10 @@
     "@types/rbush": "^3.0.0",
     "@types/seedrandom": ">=2.0.0 <4.0.0",
     "lodash": "^4.17.15",
+    "lzma-native": "6.0.0",
+    "@types/lzma-native": "4.0.0",
     "querystring": "^0.2.0",
     "rbush": "^3.0.1",
     "redux-devtools-extension": "^2.13.8"
   }
-}
+}

x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts

@@ -41,6 +41,13 @@ describe('policy details: ', () => {
               enabled: true,
               streams: [],
               config: {
+                artifact_manifest: {
+                  value: {
+                    manifest_version: 'v0',
+                    schema_version: '1.0.0',
+                    artifacts: {},
+                  },
+                },
                 policy: {
                   value: policyConfigFactory(),
                 },

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts

@@ -3,11 +3,23 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
+import { httpServerMock } from '../../../../../src/core/server/mocks';
 import { EndpointAppContextService } from './endpoint_app_context_services';
 
 describe('test endpoint app context services', () => {
-  it('should throw error if start is not called', async () => {
+  it('should throw error on getAgentService if start is not called', async () => {
     const endpointAppContextService = new EndpointAppContextService();
     expect(() => endpointAppContextService.getAgentService()).toThrow(Error);
   });
+  it('should return undefined on getManifestManager if start is not called', async () => {
+    const endpointAppContextService = new EndpointAppContextService();
+    expect(endpointAppContextService.getManifestManager()).toEqual(undefined);
+  });
+  it('should throw error on getScopedSavedObjectsClient if start is not called', async () => {
+    const endpointAppContextService = new EndpointAppContextService();
+    expect(() =>
+      endpointAppContextService.getScopedSavedObjectsClient(httpServerMock.createKibanaRequest())
+    ).toThrow(Error);
+  });
 });

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -3,14 +3,22 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import {
+  SavedObjectsServiceStart,
+  KibanaRequest,
+  SavedObjectsClientContract,
+} from 'src/core/server';
 import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
-import { handleDatasourceCreate } from './ingest_integration';
+import { getDatasourceCreateCallback } from './ingest_integration';
+import { ManifestManager } from './services/artifacts';
 
 export type EndpointAppContextServiceStartContract = Pick<
   IngestManagerStartContract,
   'agentService'
 > & {
+  manifestManager?: ManifestManager | undefined;
   registerIngestCallback: IngestManagerStartContract['registerExternalCallback'];
+  savedObjectsStart: SavedObjectsServiceStart;
 };
 
 /**
@@ -19,10 +27,20 @@ export type EndpointAppContextServiceStartContract = Pick<
  */
 export class EndpointAppContextService {
   private agentService: AgentService | undefined;
+  private manifestManager: ManifestManager | undefined;
+  private savedObjectsStart: SavedObjectsServiceStart | undefined;
 
   public start(dependencies: EndpointAppContextServiceStartContract) {
     this.agentService = dependencies.agentService;
-    dependencies.registerIngestCallback('datasourceCreate', handleDatasourceCreate);
+    this.manifestManager = dependencies.manifestManager;
+    this.savedObjectsStart = dependencies.savedObjectsStart;
+
+    if (this.manifestManager !== undefined) {
+      dependencies.registerIngestCallback(
+        'datasourceCreate',
+        getDatasourceCreateCallback(this.manifestManager)
+      );
+    }
   }
 
   public stop() {}
@@ -33,4 +51,15 @@ export class EndpointAppContextService {
     }
     return this.agentService;
   }
+
+  public getManifestManager(): ManifestManager | undefined {
+    return this.manifestManager;
+  }
+
+  public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClientContract {
+    if (!this.savedObjectsStart) {
+      throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`);
+    }
+    return this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] });
+  }
 }

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -4,46 +4,60 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { NewDatasource } from '../../../ingest_manager/common/types/models';
 import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
 import { NewPolicyData } from '../../common/endpoint/types';
-import { NewDatasource } from '../../../ingest_manager/common/types/models';
+import { ManifestManager } from './services/artifacts';
 
 /**
  * Callback to handle creation of Datasources in Ingest Manager
- * @param newDatasource
  */
-export const handleDatasourceCreate = async (
-  newDatasource: NewDatasource
-): Promise<NewDatasource> => {
-  // We only care about Endpoint datasources
-  if (newDatasource.package?.name !== 'endpoint') {
-    return newDatasource;
-  }
+export const getDatasourceCreateCallback = (
+  manifestManager: ManifestManager
+): ((newDatasource: NewDatasource) => Promise<NewDatasource>) => {
+  const handleDatasourceCreate = async (newDatasource: NewDatasource): Promise<NewDatasource> => {
+    // We only care about Endpoint datasources
+    if (newDatasource.package?.name !== 'endpoint') {
+      return newDatasource;
+    }
 
-  // We cast the type here so that any changes to the Endpoint specific data
-  // follow the types/schema expected
-  let updatedDatasource = newDatasource as NewPolicyData;
+    // We cast the type here so that any changes to the Endpoint specific data
+    // follow the types/schema expected
+    let updatedDatasource = newDatasource as NewPolicyData;
 
-  // Until we get the Default Policy Configuration in the Endpoint package,
-  // we will add it here manually at creation time.
-  // @ts-ignore
-  if (newDatasource.inputs.length === 0) {
-    updatedDatasource = {
-      ...newDatasource,
-      inputs: [
-        {
-          type: 'endpoint',
-          enabled: true,
-          streams: [],
-          config: {
-            policy: {
-              value: policyConfigFactory(),
+    const wrappedManifest = await manifestManager.refresh({ initialize: true });
+    if (wrappedManifest !== null) {
+      // Until we get the Default Policy Configuration in the Endpoint package,
+      // we will add it here manually at creation time.
+      // @ts-ignore
+      if (newDatasource.inputs.length === 0) {
+        updatedDatasource = {
+          ...newDatasource,
+          inputs: [
+            {
+              type: 'endpoint',
+              enabled: true,
+              streams: [],
+              config: {
+                artifact_manifest: {
+                  value: wrappedManifest.manifest.toEndpointFormat(),
+                },
+                policy: {
+                  value: policyConfigFactory(),
+                },
+              },
             },
-          },
-        },
-      ],
-    };
-  }
+          ],
+        };
+      }
+    }
+
+    try {
+      return updatedDatasource;
+    } finally {
+      await manifestManager.commit(wrappedManifest);
+    }
+  };
 
-  return updatedDatasource;
+  return handleDatasourceCreate;
 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ExceptionsCache } from './cache';
+
+describe('ExceptionsCache tests', () => {
+  let cache: ExceptionsCache;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    cache = new ExceptionsCache(3);
+  });
+
+  test('it should cache', async () => {
+    cache.set('test', 'body');
+    const cacheResp = cache.get('test');
+    expect(cacheResp).toEqual('body');
+  });
+
+  test('it should handle cache miss', async () => {
+    cache.set('test', 'body');
+    const cacheResp = cache.get('not test');
+    expect(cacheResp).toEqual(undefined);
+  });
+
+  test('it should handle cache eviction', async () => {
+    cache.set('1', 'a');
+    cache.set('2', 'b');
+    cache.set('3', 'c');
+    const cacheResp = cache.get('1');
+    expect(cacheResp).toEqual('a');
+
+    cache.set('4', 'd');
+    const secondResp = cache.get('1');
+    expect(secondResp).toEqual(undefined);
+    expect(cache.get('2')).toEqual('b');
+    expect(cache.get('3')).toEqual('c');
+    expect(cache.get('4')).toEqual('d');
+  });
+});

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+const DEFAULT_MAX_SIZE = 10;
+
+/**
+ * FIFO cache implementation for artifact downloads.
+ */
+export class ExceptionsCache {
+  private cache: Map<string, string>;
+  private queue: string[];
+  private maxSize: number;
+
+  constructor(maxSize: number) {
+    this.cache = new Map();
+    this.queue = [];
+    this.maxSize = maxSize || DEFAULT_MAX_SIZE;
+  }
+
+  set(id: string, body: string) {
+    if (this.queue.length + 1 > this.maxSize) {
+      const entry = this.queue.shift();
+      if (entry !== undefined) {
+        this.cache.delete(entry);
+      }
+    }
+    this.queue.push(id);
+    this.cache.set(id, body);
+  }
+
+  get(id: string): string | undefined {
+    return this.cache.get(id);
+  }
+}