[Security Solution][Endpoint] User Manifest Cleanup + Artifact Compression

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -27,8 +27,19 @@ export const getPackageConfigCreateCallback = (
     // follow the types/schema expected
     let updatedPackageConfig = newPackageConfig as NewPolicyData;
 
-    const wrappedManifest = await manifestManager.refresh({ initialize: true });
-    if (wrappedManifest !== null) {
+    // get snapshot based on exception-list-agnostic SOs
+    // with diffs from last dispatched manifest, if it exists
+    const snapshot = await manifestManager.getSnapshot({ initialize: true });
+
+    if (snapshot === null) {
+      // TODO: log error... should not be in this state
+      return updatedPackageConfig;
+    }
+
+    if (snapshot.diffs.length > 0) {
+      // create new artifacts
+      await manifestManager.syncArtifacts(snapshot, 'add');
+
       // Until we get the Default Policy Configuration in the Endpoint package,
       // we will add it here manually at creation time.
       // @ts-ignore
@@ -42,7 +53,7 @@ export const getPackageConfigCreateCallback = (
               streams: [],
               config: {
                 artifact_manifest: {
-                  value: wrappedManifest.manifest.toEndpointFormat(),
+                  value: snapshot.manifest.toEndpointFormat(),
                 },
                 policy: {
                   value: policyConfigFactory(),
@@ -57,9 +68,17 @@ export const getPackageConfigCreateCallback = (
     try {
       return updatedPackageConfig;
     } finally {
-      // TODO: confirm creation of package config
-      // then commit.
-      await manifestManager.commit(wrappedManifest);
+      if (snapshot.diffs.length > 0) {
+        const created = await manifestManager.confirmPackageConfigExists(updatedPackageConfig.name);
+        if (created) {
+          await manifestManager.commit(snapshot.manifest);
+
+          // clean up old artifacts
+          await manifestManager.syncArtifacts(snapshot, 'delete');
+        } else {
+          // TODO: log error
+        }
+      }
     }
   };
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts

@@ -88,20 +88,22 @@ export class ManifestTask {
       return;
     }
 
-    manifestManager
-      .refresh()
-      .then((wrappedManifest) => {
-        if (wrappedManifest) {
-          return manifestManager.dispatch(wrappedManifest);
-        }
-      })
-      .then((wrappedManifest) => {
-        if (wrappedManifest) {
-          return manifestManager.commit(wrappedManifest);
-        }
-      })
-      .catch((err) => {
-        this.logger.error(err);
-      });
+    try {
+      // get snapshot based on exception-list-agnostic SOs
+      // with diffs from last dispatched manifest
+      const snapshot = await manifestManager.getSnapshot();
+      if (snapshot && snapshot.diffs.length > 0) {
+        // create new artifacts
+        await manifestManager.syncArtifacts(snapshot, 'add');
+        // write to ingest-manager package config
+        await manifestManager.dispatch(snapshot.manifest);
+        // commit latest manifest state to user-artifact-manifest SO
+        await manifestManager.commit(snapshot.manifest);
+        // clean up old artifacts
+        await manifestManager.syncArtifacts(snapshot, 'delete');
+      }
+    } catch (err) {
+      this.logger.error(err);
+    }
   };
 }

x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts

@@ -10,26 +10,26 @@ import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_
 
 describe('manifest_manager', () => {
   describe('ManifestManager sanity checks', () => {
-    test('ManifestManager can refresh manifest', async () => {
+    test('ManifestManager can snapshot manifest', async () => {
       const manifestManager = getManifestManagerMock();
-      const manifestWrapper = await manifestManager.refresh();
-      expect(manifestWrapper!.diffs).toEqual([
+      const snapshot = await manifestManager.getSnapshot();
+      expect(snapshot!.diffs).toEqual([
         {
           id:
             'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49',
           type: 'add',
         },
       ]);
-      expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest);
+      expect(snapshot!.manifest).toBeInstanceOf(Manifest);
     });
 
     test('ManifestManager can dispatch manifest', async () => {
       const packageConfigService = getPackageConfigServiceMock();
       const manifestManager = getManifestManagerMock({ packageConfigService });
-      const manifestWrapperRefresh = await manifestManager.refresh();
-      const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh);
-      expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch);
-      const entries = manifestWrapperDispatch!.manifest.getEntries();
+      const snapshot = await manifestManager.getSnapshot();
+      const dispatched = await manifestManager.dispatch(snapshot!.manifest);
+      expect(dispatched).toEqual(true);
+      const entries = snapshot!.manifest.getEntries();
       const artifact = Object.values(entries)[0].getArtifact();
       expect(
         packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value
@@ -56,15 +56,21 @@ describe('manifest_manager', () => {
         savedObjectsClient,
       });
 
-      const manifestWrapperRefresh = await manifestManager.refresh();
-      const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh);
+      const snapshot = await manifestManager.getSnapshot();
+      await manifestManager.syncArtifacts(snapshot!, 'add');
+
       const diff = {
         id: 'abcd',
         type: 'delete',
       };
-      manifestWrapperDispatch!.diffs.push(diff);
+      snapshot!.diffs.push(diff);
+
+      const dispatched = await manifestManager.dispatch(snapshot!.manifest);
+      expect(dispatched).toEqual(true);
+
+      await manifestManager.commit(snapshot!.manifest);
 
-      await manifestManager.commit(manifestWrapperDispatch);
+      await manifestManager.syncArtifacts(snapshot!, 'delete');
 
       // created new artifact
       expect(savedObjectsClient.create.mock.calls[0][0]).toEqual(