Use worker threads when running lint --serially (#607)

.changeset/metal-rings-listen.md

@@ -0,0 +1,7 @@
+---
+"skuba": patch
+---
+
+lint: Use worker threads when running `--serial`ly
+
+This aims to reduce the memory footprint of `skuba lint --serial`. ESLint and Prettier are now run in worker threads so their memory can be more readily freed on thread exit.

src/cli/lint.int.test.ts

@@ -106,8 +106,8 @@ test.each`
   await prepareTempDirectory(baseDir, tempDir);
 
   await expect(
-    // Force `--serial` as worker threads can't run in a TypeScript context.
-    lint([...args, '--serial'], tscOutputStream),
+    // Disable worker threads as they can't run in a TypeScript context.
+    lint(args, tscOutputStream, false),
   ).resolves.toBeUndefined();
 
   const tempDirRegex = new RegExp(tempDir, 'g');

src/cli/lint/external.ts

@@ -39,19 +39,42 @@ const lintConcurrently = async ({ tscOutputStream, ...input }: Input) => {
   return { eslint, prettier, tscOk };
 };
 
-const lintSerially = async (input: Input) => {
+/**
+ * Run linting tools `--serial`ly for resource-constrained environments.
+ *
+ * Note that we still run ESLint and Prettier in worker threads as a
+ * counterintuitive optimisation. Memory can be more readily freed on worker
+ * thread exit, which isn't as easy with a monolithic main thread.
+ */
+const lintSerially = async ({ tscOutputStream, ...input }: Input) => {
+  const eslint = await runESLintInWorkerThread(input);
+  const prettier = await runPrettierInWorkerThread(input);
+  const tscOk = await runTscInNewProcess({ ...input, tscOutputStream });
+
+  return { eslint, prettier, tscOk };
+};
+
+const lintSeriallyWithoutWorkerThreads = async (input: Input) => {
   const eslint = await runESLintInCurrentThread(input);
   const prettier = await runPrettierInCurrentThread(input);
   const tscOk = await runTscInNewProcess(input);
 
   return { eslint, prettier, tscOk };
 };
 
-export const externalLint = async (input: Input) => {
+const selectLintFunction = (input: Input) => {
+  if (!input.workerThreads) {
+    return lintSeriallyWithoutWorkerThreads;
+  }
+
   // `--debug` implies `--serial`.
   const isSerial = input.debug || input.serial;
 
-  const lint = isSerial ? lintSerially : lintConcurrently;
+  return isSerial ? lintSerially : lintConcurrently;
+};
+
+export const externalLint = async (input: Input) => {
+  const lint = selectLintFunction(input);
 
   const tscOutputStream = new StreamInterceptor();
   tscOutputStream.pipe(input.tscOutputStream ?? process.stdout);

src/cli/lint/index.ts

@@ -7,11 +7,13 @@ import type { Input } from './types';
 export const lint = async (
   args = process.argv,
   tscOutputStream: NodeJS.WritableStream | undefined = undefined,
+  workerThreads = true,
 ) => {
   const opts: Input = {
     debug: hasDebugFlag(args),
     serial: hasSerialFlag(args),
     tscOutputStream,
+    workerThreads,
   };
 
   await externalLint(opts);

src/cli/lint/types.ts

@@ -22,4 +22,14 @@ export interface Input {
    * Defaults to `process.stdout`.
    */
   tscOutputStream?: NodeJS.WritableStream;
+
+  /**
+   * Whether to allow usage of Node.js worker threads.
+   *
+   * This may be set to `false` when there is a worker thread incompatibility,
+   * such as calling in from a TypeScript context in our Jest tests.
+   *
+   * Defaults to `true`.
+   */
+  workerThreads?: boolean;
 }