From f482307cf66eb0f2348e213e63d46e16c7d41ab2 Mon Sep 17 00:00:00 2001 From: Evan Bacon <9664363+EvanBacon@users.noreply.github.com> Date: Tue, 24 Aug 2021 20:31:04 -0600 Subject: [PATCH 1/4] Use more optimal suffix-set format --- packages/jest-haste-map/src/crawlers/watchman.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 9b20f42e5597..7bbdf753073c 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -67,7 +67,8 @@ export = async function watchmanCrawl(options: CrawlerOptions): Promise<{ const defaultWatchExpression = [ 'allof', ['type', 'f'], - ['anyof', ...extensions.map(extension => ['suffix', extension])], + // suffix-set https://facebook.github.io/watchman/docs/expr/suffix.html#suffix-set + ['suffix', extensions] ]; const clocks = data.clocks; const client = new watchman.Client(); From 4ca816fca8309d0a89bf9082cf35243f70cea317 Mon Sep 17 00:00:00 2001 From: Evan Bacon <9664363+EvanBacon@users.noreply.github.com> Date: Wed, 25 Aug 2021 12:36:33 -0600 Subject: [PATCH 2/4] Update watchman.ts --- .../jest-haste-map/src/crawlers/watchman.ts | 60 +++++++++++++++++-- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index 7bbdf753073c..b7e63ab51eeb 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -24,6 +24,13 @@ type WatchmanListCapabilitiesResponse = { capabilities: Array; }; +type WatchmanCapabilityCheckResponse = { + // { 'suffix-set': true } + capabilities: Record; + // '2021.06.07.00' + version: string; +}; + type WatchmanWatchProjectResponse = { watch: string; relative_path: string; @@ -57,6 +64,32 @@ function WatchmanError(error: Error): Error { return error; } +/** + * Wrap watchman capabilityCheck method as a promise. + * + * @param client watchman client + * @param caps capabilities to verify + * @returns a promise resolving to a list of verified capabilities + */ +async function capabilityCheck( + client: watchman.Client, + caps: Partial, +): Promise { + return new Promise((resolve, reject) => { + client.capabilityCheck( + // @ts-expect-error: incorrectly typed + caps, + (error, response) => { + if (error) { + reject(error); + } else { + resolve(response); + } + }, + ); + }); +} + export = async function watchmanCrawl(options: CrawlerOptions): Promise<{ changedFiles?: FileData; removedFiles: FileData; @@ -64,15 +97,30 @@ export = async function watchmanCrawl(options: CrawlerOptions): Promise<{ }> { const fields = ['name', 'exists', 'mtime_ms', 'size']; const {data, extensions, ignore, rootDir, roots} = options; - const defaultWatchExpression = [ - 'allof', - ['type', 'f'], - // suffix-set https://facebook.github.io/watchman/docs/expr/suffix.html#suffix-set - ['suffix', extensions] - ]; + const defaultWatchExpression: Array = ['allof', ['type', 'f']]; const clocks = data.clocks; const client = new watchman.Client(); + // https://facebook.github.io/watchman/docs/capabilities.html + // Check adds about ~28ms + const capabilities = await capabilityCheck(client, { + // If a required capability is missing then an error will be thrown, + // we don't need this assertion, so using optional instead. + optional: ['suffix-set'], + }); + + if (capabilities?.capabilities['suffix-set']) { + // If available, use the optimized `suffix-set` operation: + // https://facebook.github.io/watchman/docs/expr/suffix.html#suffix-set + defaultWatchExpression.push(['suffix', extensions]); + } else { + // Otherwise use the older and less optimal suffix tuple array + defaultWatchExpression.push([ + 'anyof', + ...extensions.map(extension => ['suffix', extension]), + ]); + } + let clientError; client.on('error', error => (clientError = WatchmanError(error))); From b338e4739339de5ca7aac83f7f3f4e06dd52be7e Mon Sep 17 00:00:00 2001 From: Evan Bacon <9664363+EvanBacon@users.noreply.github.com> Date: Thu, 26 Aug 2021 19:10:49 -0600 Subject: [PATCH 3/4] Changelog entry and updated tests --- CHANGELOG.md | 1 + .../src/crawlers/__tests__/watchman.test.js | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 732ed0c47d2a..cf43d07d99e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +- `[jest-haste-map]` Use watchman suffix-set option for faster file indexing. ([#11784](https://github.com/facebook/jest/pull/11784)) - `[jest-cli]` Adds a new config options `snapshotFormat` which offers a way to override any of the formatting settings which come with [pretty-format](https://www.npmjs.com/package/pretty-format#usage-with-options). ([#11654](https://github.com/facebook/jest/pull/11654)) ### Fixes diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index 337620773122..b8ae9a609acf 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -13,6 +13,14 @@ const path = require('path'); jest.mock('fb-watchman', () => { const normalizePathSep = require('../../lib/normalizePathSep').default; const Client = jest.fn(); + Client.prototype.capabilityCheck = jest.fn((args, callback) => + setImmediate(() => { + callback(null, { + capabilities: {'suffix-set': true}, + version: '2021.06.07.00', + }); + }), + ); Client.prototype.command = jest.fn((args, callback) => setImmediate(() => { const path = args[1] ? normalizePathSep(args[1]) : undefined; From 9fd993412c1aa43a336d60b897c6d2c370963574 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 27 Aug 2021 10:30:13 +0200 Subject: [PATCH 4/4] update tests --- .../jest-haste-map/src/crawlers/__tests__/watchman.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index b8ae9a609acf..4fd59bc729c9 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -153,7 +153,7 @@ describe('watchman watch', () => { expect(query[2].expression).toEqual([ 'allof', ['type', 'f'], - ['anyof', ['suffix', 'js'], ['suffix', 'json']], + ['suffix', ['js', 'json']], ['anyof', ['dirname', 'fruits'], ['dirname', 'vegetables']], ]); @@ -496,7 +496,7 @@ describe('watchman watch', () => { expect(query[2].expression).toEqual([ 'allof', ['type', 'f'], - ['anyof', ['suffix', 'js'], ['suffix', 'json']], + ['suffix', ['js', 'json']], ]); expect(query[2].fields).toEqual(['name', 'exists', 'mtime_ms', 'size']);