feat(cache): adds option to compress the cache file (#860)

## Description

- adds an option to compress the cache file
- 🏕️ some updates in the cache typing

## Motivation and Context

The cache file can become quite large. The cache file is also _very_
compressible (both gzip and brotli reduce the file size to 10% of the
original _or less_). My original hunch was that as less i/o is needed
this might constitute a net performance gain for large enough code
bases. I haven't been able to substantiate that with evidence, though.

However, I decided to still let this in because:
- when you switch compression on, it (only) adds a few milliseconds to
the total execution time
- the resulting file is so much smaller, that the trade-off of the few
extra milliseconds might be worth it for other reasons

## How Has This Been Tested?

- [x] green ci
- [x] additional automated tests

## Types of changes

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] Documentation only change
- [ ] Refactor (non-breaking change which fixes an issue without
changing functionality)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)

.dependency-cruiser.json

@@ -500,7 +500,8 @@
     },
     "progress": { "type": "performance-log", "maximumLevel": 60 },
     "cache": {
-      "strategy": "metadata"
+      "strategy": "metadata",
+      "compress": true
     }
   }
 }

doc/options-reference.md

@@ -1644,7 +1644,12 @@ The long form:
       // background) or 'content' (which will look at file content (hashes),
       // is slower than 'metadata' and is a bleeding edge feature as of
       // version 12.5.0)
-      strategy: "metadata"
+      strategy: "metadata",
+      // whether or not to compress the cache file. Switching this to true
+      // will make dependency-cruiser a few milliseconds slower over all.
+      // The resulting cache file will be 80-90% smaller though.
+      // Defaults to false (don't compress)
+      compress: false
     }
     // ...
   }
@@ -1668,7 +1673,7 @@ will interpret that as the cache folder.
 }
 ```
 
-If you don't want to use caching you cah leave the cache option out altogether or
+If you don't want to use caching you can leave the cache option out altogether or
 use `cache: false`.
 
 As with most settings the command line option of the same name takes

src/cache/cache.mjs

@@ -1,6 +1,11 @@
 // @ts-check
 import { readFile, mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
+import {
+  brotliCompressSync,
+  brotliDecompressSync,
+  constants as zlibConstants,
+} from "node:zlib";
 import { optionsAreCompatible } from "./options-compatible.mjs";
 import MetadataStrategy from "./metadata-strategy.mjs";
 import ContentStrategy from "./content-strategy.mjs";
@@ -10,17 +15,31 @@ import { scannableExtensions } from "#extract/transpile/meta.mjs";
 import { bus } from "#utl/bus.mjs";
 
 const CACHE_FILE_NAME = "cache.json";
+const EMPTY_CACHE = {
+  modules: [],
+  summary: {
+    error: 0,
+    warn: 0,
+    info: 0,
+    ignore: 0,
+    totalCruised: 0,
+    violations: [],
+    optionsUsed: {},
+  },
+};
 
 export default class Cache {
   /**
    * @param {import("../../types/cache-options.js").cacheStrategyType=} pCacheStrategy
+   * @param {boolean=} pCompress
    */
-  constructor(pCacheStrategy) {
+  constructor(pCacheStrategy, pCompress) {
     this.revisionData = null;
     this.cacheStrategy =
       pCacheStrategy === "content"
         ? new ContentStrategy()
         : new MetadataStrategy();
+    this.compress = pCompress ?? false;
   }
 
   /**
@@ -49,6 +68,8 @@ export default class Cache {
         this.revisionData,
       ) &&
       optionsAreCompatible(
+        // @ts-expect-error ts(2345) - it's indeed not strict cruise options,
+        // but it will do for now (_it works_)
         pCachedCruiseResult.summary.optionsUsed,
         pCruiseOptions,
       )
@@ -61,14 +82,55 @@ export default class Cache {
    */
   async read(pCacheFolder) {
     try {
-      return JSON.parse(
-        await readFile(join(pCacheFolder, CACHE_FILE_NAME), "utf8"),
-      );
+      let lPayload = "";
+      if (this.compress === true) {
+        const lCompressedPayload = await readFile(
+          join(pCacheFolder, CACHE_FILE_NAME),
+        );
+        const lPayloadAsBuffer = brotliDecompressSync(lCompressedPayload);
+        lPayload = lPayloadAsBuffer.toString("utf8");
+      } else {
+        lPayload = await readFile(join(pCacheFolder, CACHE_FILE_NAME), "utf8");
+      }
+      return JSON.parse(lPayload);
     } catch (pError) {
-      return { modules: [], summary: {} };
+      return EMPTY_CACHE;
     }
   }
 
+  /**
+   * @param {string} pPayload
+   * @param {boolean} pCompress
+   * @return {Buffer|string}
+   */
+  #compact(pPayload, pCompress) {
+    if (pCompress) {
+      /**
+       * we landed on brotli with BROTLI_MIN_QUALITY because:
+       * - even with BROTLI_MIN_QUALITY it compresses better than gzip
+       *   (regardless of compression level)
+       * - at BROTLI_MIN_QUALITY it's faster than gzip
+       * - BROTLI_MAX_QUALITY gives a bit better compression but is _much_
+       *   slower than even gzip
+       *
+       * In our situation the sync version is significantly faster than the
+       * async version + zlib functions need to be promisified before they
+       * can be used in promises, which will add the to the execution time
+       * as well.
+       *
+       * As sync or async doesn't _really_
+       * matter for the cli, we're using the sync version here.
+       */
+      return brotliCompressSync(pPayload, {
+        params: {
+          [zlibConstants.BROTLI_PARAM_QUALITY]:
+            zlibConstants.BROTLI_MIN_QUALITY,
+        },
+      });
+    }
+    return pPayload;
+  }
+
   /**
    * @param {string} pCacheFolder
    * @param {import("../../types/dependency-cruiser.js").ICruiseResult} pCruiseResult
@@ -78,15 +140,16 @@ export default class Cache {
     const lRevisionData = pRevisionData ?? this.revisionData;
 
     await mkdir(pCacheFolder, { recursive: true });
-    await writeFile(
-      join(pCacheFolder, CACHE_FILE_NAME),
-      JSON.stringify(
-        this.cacheStrategy.prepareRevisionDataForSaving(
-          pCruiseResult,
-          lRevisionData,
-        ),
+    const lUncompressedPayload = JSON.stringify(
+      this.cacheStrategy.prepareRevisionDataForSaving(
+        pCruiseResult,
+        lRevisionData,
       ),
-      "utf8",
     );
+    let lPayload = this.#compact(lUncompressedPayload, this.compress);
+
+    // relying on writeFile defaults to 'do the right thing' (i.e. utf8
+    // when the payload is a string; raw buffer otherwise)
+    await writeFile(join(pCacheFolder, CACHE_FILE_NAME), lPayload);
   }
 }

src/cache/content-strategy.mjs

@@ -36,8 +36,8 @@ function refreshChanges(pChanges, pModules) {
       !pModules.some(
         (pModule) =>
           pModule.source === pChange.name &&
-          pModule.checksum === pChange.checksum
-      )
+          pModule.checksum === pChange.checksum,
+      ),
   );
 }
 
@@ -48,10 +48,10 @@ export default class ContentStrategy {
    * @param {import("../../types/strict-options.js").IStrictCruiseOptions} pCruiseOptions
    * @param {Object} pOptions
    * @param {Set<string>} pOptions.extensions
-   * @param {Set<import("watskeburt").changeTypeType>} pOptions.interestingChangeTypes
+   * @param {Set<import("watskeburt").changeTypeType>=} pOptions.interestingChangeTypes?
    * @param {string=} pOptions.baseDir
    * @param {typeof findContentChanges=} pOptions.diffListFn
-   * @param {typeof import('watskeburt').getSHASync=} pOptions.checksumFn
+   * @param {typeof import('watskeburt').getSHA=} pOptions.checksumFn
    * @returns {import("../../types/dependency-cruiser.js").IRevisionData}
    */
   getRevisionData(pDirectory, pCachedCruiseResult, pCruiseOptions, pOptions) {
@@ -89,19 +89,19 @@ export default class ContentStrategy {
         pExistingRevisionData.SHA1 === pNewRevisionData.SHA1 &&
         isDeepStrictEqual(
           pExistingRevisionData.changes,
-          pNewRevisionData.changes
-        )
+          pNewRevisionData.changes,
+        ),
     );
   }
 
   /**
    * @param {import("../../types/dependency-cruiser.js").ICruiseResult} pCruiseResult
-   * @param {import("../../types/dependency-cruiser.js").IRevisionData} pRevisionData
+   * @param {import("../../types/dependency-cruiser.js").IRevisionData=} pRevisionData
    * @returns {import("../../types/dependency-cruiser.js").ICruiseResult}
    */
   prepareRevisionDataForSaving(pCruiseResult, pRevisionData) {
     const lModulesWithCheckSum = pCruiseResult.modules.map(
-      addCheckSumToModule(pCruiseResult.summary.optionsUsed.baseDir)
+      addCheckSumToModule(pCruiseResult.summary.optionsUsed.baseDir),
     );
     const lRevisionData = {
       ...pRevisionData,

src/cache/find-content-changes.mjs

@@ -69,6 +69,8 @@ function diffCachedModuleAgainstFileSet(
  * @param {Object} pOptions
  * @param {Set<string>} pOptions.extensions
  * @param {string} pOptions.baseDir
+ * @param {import("../../types/strict-filter-types").IStrictExcludeType} pOptions.exclude
+ * @param {import("../../types/strict-filter-types").IStrictIncludeOnlyType=} pOptions.includeOnly
  * @returns {import("../..").IRevisionChange[]}
  */
 export default function findContentChanges(

src/cache/helpers.mjs

@@ -4,6 +4,7 @@ import { readFileSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import { extname } from "node:path";
 import memoize from "lodash/memoize.js";
+// @ts-expect-error ts(2307) - the ts compiler is not privy to the existence of #imports in package.json
 import { filenameMatchesPattern } from "#graph-utl/match-facade.mjs";
 
 /**
@@ -62,7 +63,7 @@ export function addCheckSumToChangeSync(pChange) {
 
 /**
  *
- * @param {import("../../types/strict-filter-types.js").IStrictExcludeType} pExcludeOption
+ * @param {import("../../types/filter-types.js").IExcludeType} pExcludeOption
  * @returns {(pFileName: string) => boolean}
  */
 export function excludeFilter(pExcludeOption) {
@@ -75,7 +76,7 @@ export function excludeFilter(pExcludeOption) {
 }
 
 /**
- * @param {import("../../types/strict-filter-types.js").IStrictIncludeOnlyType} pIncludeOnlyFilter
+ * @param {import("../../types/strict-filter-types.js").IStrictIncludeOnlyType=} pIncludeOnlyFilter
  * @returns {(pFileName: string) => boolean}
  */
 export function includeOnlyFilter(pIncludeOnlyFilter) {
@@ -122,7 +123,7 @@ const DEFAULT_INTERESTING_CHANGE_TYPES = new Set([
 ]);
 
 /**
- * @param {Set<import("watskeburt").changeTypeType>} pInterestingChangeTypes
+ * @param {Set<import("watskeburt").changeTypeType>=} pInterestingChangeTypes
  * @returns {(pChange: import("watskeburt").IChange) => boolean}
  */
 export function isInterestingChangeType(pInterestingChangeTypes) {

src/cache/metadata-strategy.mjs

@@ -18,7 +18,7 @@ export default class MetaDataStrategy {
    * @param {import("../../types/cruise-result.js").ICruiseResult} _pCachedCruiseResult
    * @param {Object} pOptions
    * @param {Set<string>} pOptions.extensions
-   * @param {Set<import("watskeburt").changeTypeType>} pOptions.interestingChangeTypes
+   * @param {Set<import("watskeburt").changeTypeType>=} pOptions.interestingChangeTypes
    * @param {typeof getSHA=} pOptions.shaRetrievalFn
    * @param {typeof list=} pOptions.diffListFn
    * @param {typeof addCheckSumToChangeSync=} pOptions.checksumFn
@@ -63,15 +63,19 @@ export default class MetaDataStrategy {
   }
 
   /**
-   * @param {import("../../types/dependency-cruiser.js").IRevisionData} pExistingRevisionData
-   * @param {import("../../types/dependency-cruiser.js").IRevisionData} pNewRevisionData
+   * @param {import("../../types/dependency-cruiser.js").IRevisionData=} pExistingRevisionData
+   * @param {import("../../types/dependency-cruiser.js").IRevisionData=} pNewRevisionData
    * @returns {boolean}
    */
   revisionDataEqual(pExistingRevisionData, pNewRevisionData) {
     return (
       Boolean(pExistingRevisionData) &&
       Boolean(pNewRevisionData) &&
+      // @ts-expect-error ts(18048) - tsc complains pExistingRevisionData &
+      // pNewRevisionData can be undefined, but it should probably get a course
+      // in reading typescript as we've just checked this.
       pExistingRevisionData.SHA1 === pNewRevisionData.SHA1 &&
+      // @ts-expect-error ts(18048)
       isDeepStrictEqual(pExistingRevisionData.changes, pNewRevisionData.changes)
     );
   }

src/main/cruise.mjs

@@ -33,7 +33,10 @@ export default async function cruise(
     );
 
     const { default: Cache } = await import("#cache/cache.mjs");
-    lCache = new Cache(lCruiseOptions.cache.strategy);
+    lCache = new Cache(
+      lCruiseOptions.cache.strategy,
+      lCruiseOptions.cache.compress,
+    );
     const lCachedResults = await lCache.read(lCruiseOptions.cache.folder);
 
     if (await lCache.canServeFromCache(lCruiseOptions, lCachedResults)) {