[Security Solution][Endpoint] User Manifest Cleanup + Artifact Compression

x-pack/plugins/security_solution/kibana.json

@@ -11,7 +11,6 @@
     "embeddable",
     "features",
     "home",
-    "ingestManager",
     "taskManager",
     "inspector",
     "licensing",
@@ -21,6 +20,7 @@
   ],
   "optionalPlugins": [
     "encryptedSavedObjects",
+    "ingestManager",
     "ml",
     "newsfeed",
     "security",

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts

@@ -8,11 +8,11 @@ import { httpServerMock } from '../../../../../src/core/server/mocks';
 import { EndpointAppContextService } from './endpoint_app_context_services';
 
 describe('test endpoint app context services', () => {
-  it('should throw error on getAgentService if start is not called', async () => {
+  it('should return undefined on getAgentService if dependencies are not enabled', async () => {
     const endpointAppContextService = new EndpointAppContextService();
     expect(() => endpointAppContextService.getAgentService()).toThrow(Error);
   });
-  it('should return undefined on getManifestManager if start is not called', async () => {
+  it('should return undefined on getManifestManager if dependencies are not enabled', async () => {
     const endpointAppContextService = new EndpointAppContextService();
     expect(endpointAppContextService.getManifestManager()).toEqual(undefined);
   });

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -12,12 +12,11 @@ import { AgentService, IngestManagerStartContract } from '../../../ingest_manage
 import { getPackageConfigCreateCallback } from './ingest_integration';
 import { ManifestManager } from './services/artifacts';
 
-export type EndpointAppContextServiceStartContract = Pick<
-  IngestManagerStartContract,
-  'agentService'
+export type EndpointAppContextServiceStartContract = Partial<
+  Pick<IngestManagerStartContract, 'agentService'>
 > & {
-  manifestManager?: ManifestManager | undefined;
-  registerIngestCallback: IngestManagerStartContract['registerExternalCallback'];
+  manifestManager?: ManifestManager;
+  registerIngestCallback?: IngestManagerStartContract['registerExternalCallback'];
   savedObjectsStart: SavedObjectsServiceStart;
 };
 
@@ -35,7 +34,7 @@ export class EndpointAppContextService {
     this.manifestManager = dependencies.manifestManager;
     this.savedObjectsStart = dependencies.savedObjectsStart;
 
-    if (this.manifestManager !== undefined) {
+    if (this.manifestManager && dependencies.registerIngestCallback) {
       dependencies.registerIngestCallback(
         'packageConfigCreate',
         getPackageConfigCreateCallback(this.manifestManager)
@@ -45,10 +44,7 @@ export class EndpointAppContextService {
 
   public stop() {}
 
-  public getAgentService(): AgentService {
-    if (!this.agentService) {
-      throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`);
-    }
+  public getAgentService(): AgentService | undefined {
     return this.agentService;
   }
 

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -27,8 +27,19 @@ export const getPackageConfigCreateCallback = (
     // follow the types/schema expected
     let updatedPackageConfig = newPackageConfig as NewPolicyData;
 
-    const wrappedManifest = await manifestManager.refresh({ initialize: true });
-    if (wrappedManifest !== null) {
+    // get snapshot based on exception-list-agnostic SOs
+    // with diffs from last dispatched manifest, if it exists
+    const snapshot = await manifestManager.getSnapshot({ initialize: true });
+
+    if (snapshot === null) {
+      // TODO: log error... should not be in this state
+      return updatedPackageConfig;
+    }
+
+    if (snapshot.diffs.length > 0) {
+      // create new artifacts
+      await manifestManager.syncArtifacts(snapshot, 'add');
+
       // Until we get the Default Policy Configuration in the Endpoint package,
       // we will add it here manually at creation time.
       // @ts-ignore
@@ -42,7 +53,7 @@ export const getPackageConfigCreateCallback = (
               streams: [],
               config: {
                 artifact_manifest: {
-                  value: wrappedManifest.manifest.toEndpointFormat(),
+                  value: snapshot.manifest.toEndpointFormat(),
                 },
                 policy: {
                   value: policyConfigFactory(),
@@ -57,9 +68,18 @@ export const getPackageConfigCreateCallback = (
     try {
       return updatedPackageConfig;
     } finally {
-      // TODO: confirm creation of package config
-      // then commit.
-      await manifestManager.commit(wrappedManifest);
+      if (snapshot.diffs.length > 0) {
+        // const created = await manifestManager.confirmPackageConfigExists(updatedPackageConfig.name);
+        const created = true;
+        if (created) {
+          await manifestManager.commit(snapshot.manifest);
+
+          // clean up old artifacts
+          await manifestManager.syncArtifacts(snapshot, 'delete');
+        } else {
+          // TODO: log error
+        }
+      }
     }
   };
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts

@@ -10,7 +10,7 @@ const DEFAULT_MAX_SIZE = 10;
  * FIFO cache implementation for artifact downloads.
  */
 export class ExceptionsCache {
-  private cache: Map<string, string>;
+  private cache: Map<string, Buffer>;
   private queue: string[];
   private maxSize: number;
 
@@ -20,7 +20,7 @@ export class ExceptionsCache {
     this.maxSize = maxSize || DEFAULT_MAX_SIZE;
   }
 
-  set(id: string, body: string) {
+  set(id: string, body: Buffer) {
     if (this.queue.length + 1 > this.maxSize) {
       const entry = this.queue.shift();
       if (entry !== undefined) {
@@ -31,7 +31,7 @@ export class ExceptionsCache {
     this.cache.set(id, body);
   }
 
-  get(id: string): string | undefined {
+  get(id: string): Buffer | undefined {
     return this.cache.get(id);
   }
 }

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -5,6 +5,7 @@
  */
 
 import { createHash } from 'crypto';
+import { deflate } from 'zlib';
 import { validate } from '../../../../common/validate';
 
 import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries';
@@ -32,6 +33,7 @@ export async function buildArtifact(
   const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions));
   const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex');
 
+  // Keep compression info empty in case its a duplicate. Lazily compress before committing if needed.
   return {
     identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
     compressionAlgorithm: 'none',
@@ -170,3 +172,15 @@ function translateEntry(
     }
   }
 }
+
+export function compressExceptionList(buffer: Buffer): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    deflate(buffer, function (err, buf) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(buf);
+      }
+    });
+  });
+}

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts

@@ -88,20 +88,22 @@ export class ManifestTask {
       return;
     }
 
-    manifestManager
-      .refresh()
-      .then((wrappedManifest) => {
-        if (wrappedManifest) {
-          return manifestManager.dispatch(wrappedManifest);
-        }
-      })
-      .then((wrappedManifest) => {
-        if (wrappedManifest) {
-          return manifestManager.commit(wrappedManifest);
-        }
-      })
-      .catch((err) => {
-        this.logger.error(err);
-      });
+    try {
+      // get snapshot based on exception-list-agnostic SOs
+      // with diffs from last dispatched manifest
+      const snapshot = await manifestManager.getSnapshot();
+      if (snapshot && snapshot.diffs.length > 0) {
+        // create new artifacts
+        await manifestManager.syncArtifacts(snapshot, 'add');
+        // write to ingest-manager package config
+        await manifestManager.dispatch(snapshot.manifest);
+        // commit latest manifest state to user-artifact-manifest SO
+        await manifestManager.commit(snapshot.manifest);
+        // clean up old artifacts
+        await manifestManager.syncArtifacts(snapshot, 'delete');
+      }
+    } catch (err) {
+      this.logger.error(err);
+    }
   };
 }

x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts

@@ -77,7 +77,6 @@ describe('test alerts route', () => {
   let mockScopedClient: jest.Mocked<ILegacyScopedClusterClient>;
   let mockSavedObjectClient: jest.Mocked<SavedObjectsClientContract>;
   let mockResponse: jest.Mocked<KibanaResponseFactory>;
-  // @ts-ignore
   let routeConfig: RouteConfig<unknown, unknown, unknown, never>;
   let routeHandler: RequestHandler<unknown, unknown, unknown>;
   let endpointAppContextService: EndpointAppContextService;
@@ -98,8 +97,9 @@ describe('test alerts route', () => {
     // The authentication with the Fleet Plugin needs a separate scoped SO Client
     ingestSavedObjectClient = savedObjectsClientMock.create();
     ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse));
-    // @ts-ignore
-    startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient);
+    (startContract.savedObjectsStart.getScopedClient as jest.Mock).mockReturnValue(
+      ingestSavedObjectClient
+    );
     endpointAppContextService.start(startContract);
 
     registerDownloadExceptionListRoute(
@@ -147,6 +147,8 @@ describe('test alerts route', () => {
       path.startsWith('/api/endpoint/artifacts/download')
     )!;
 
+    expect(routeConfig.options).toEqual({});
+
     await routeHandler(
       ({
         core: {
@@ -160,8 +162,8 @@ describe('test alerts route', () => {
     );
 
     const expectedHeaders = {
-      'content-encoding': 'application/json',
-      'content-disposition': `attachment; filename=${mockArtifactName}.json`,
+      'content-encoding': 'identity',
+      'content-disposition': `attachment; filename=${mockArtifactName}.zz`,
     };
 
     expect(mockResponse.ok).toBeCalled();
@@ -217,7 +219,7 @@ describe('test alerts route', () => {
     // Add to the download cache
     const mockArtifact = expectedEndpointExceptions;
     const cacheKey = `${mockArtifactName}-${mockSha}`;
-    cache.set(cacheKey, JSON.stringify(mockArtifact));
+    cache.set(cacheKey, Buffer.from(JSON.stringify(mockArtifact))); // TODO: add compression here
 
     [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) =>
       path.startsWith('/api/endpoint/artifacts/download')

x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts

@@ -45,7 +45,6 @@ export function registerDownloadExceptionListRoute(
       },
       options: { tags: [] },
     },
-    // @ts-ignore
     async (context, req, res) => {
       let scopedSOClient: SavedObjectsClientContract;
       const logger = endpointContext.logFactory.get('download_exception_list');
@@ -62,12 +61,12 @@ export function registerDownloadExceptionListRoute(
         }
       }
 
-      const buildAndValidateResponse = (artName: string, body: string): IKibanaResponse => {
+      const buildAndValidateResponse = (artName: string, body: Buffer): IKibanaResponse => {
         const artifact: HttpResponseOptions = {
           body,
           headers: {
-            'content-encoding': 'application/json',
-            'content-disposition': `attachment; filename=${artName}.json`,
+            'content-encoding': 'identity',
+            'content-disposition': `attachment; filename=${artName}.zz`,
           },
         };
 
@@ -90,7 +89,7 @@ export function registerDownloadExceptionListRoute(
         return scopedSOClient
           .get<InternalArtifactSchema>(ArtifactConstants.SAVED_OBJECT_TYPE, id)
           .then((artifact: SavedObject<InternalArtifactSchema>) => {
-            const body = Buffer.from(artifact.attributes.body, 'base64').toString();
+            const body = Buffer.from(artifact.attributes.body, 'base64');
             cache.set(id, body);
             return buildAndValidateResponse(artifact.attributes.identifier, body);
           })