From f19d1db8817e88ef8942e248c9293de43934c707 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli Date: Fri, 20 Oct 2023 17:20:00 -0400 Subject: [PATCH] feat(core): remove parcel/watcher (#19751) --- packages/nx/package.json | 1 - packages/nx/src/daemon/client/client.ts | 6 - .../daemon/server/outputs-tracking.spec.ts | 15 +- .../nx/src/daemon/server/outputs-tracking.ts | 10 +- ...project-graph-incremental-recomputation.ts | 29 +--- packages/nx/src/daemon/server/server.ts | 139 ++++++------------ .../nx/src/daemon/server/shutdown-utils.ts | 44 ------ packages/nx/src/daemon/server/watcher.ts | 127 +--------------- 8 files changed, 61 insertions(+), 310 deletions(-) diff --git a/packages/nx/package.json b/packages/nx/package.json index 3c002f4cb203e..a3ea112f90d2a 100644 --- a/packages/nx/package.json +++ b/packages/nx/package.json @@ -33,7 +33,6 @@ }, "homepage": "https://nx.dev", "dependencies": { - "@parcel/watcher": "2.0.4", "@yarnpkg/lockfile": "^1.1.0", "@yarnpkg/parsers": "3.0.0-rc.46", "@zkochan/js-yaml": "0.0.6", diff --git a/packages/nx/src/daemon/client/client.ts b/packages/nx/src/daemon/client/client.ts index d31f3349496c4..7a4a85372e00a 100644 --- a/packages/nx/src/daemon/client/client.ts +++ b/packages/nx/src/daemon/client/client.ts @@ -366,12 +366,6 @@ export class DaemonClient { this._out = await open(DAEMON_OUTPUT_LOG_FILE, 'a'); this._err = await open(DAEMON_OUTPUT_LOG_FILE, 'a'); - if (this.nxJson.tasksRunnerOptions?.default?.options?.useParcelWatcher) { - DAEMON_ENV_SETTINGS['NX_NATIVE_WATCHER'] = 'false'; - } else { - DAEMON_ENV_SETTINGS['NX_NATIVE_WATCHER'] = 'true'; - } - const backgroundProcess = spawn( process.execPath, [join(__dirname, '../server/start.js')], diff --git a/packages/nx/src/daemon/server/outputs-tracking.spec.ts b/packages/nx/src/daemon/server/outputs-tracking.spec.ts index cca4b91699dca..3854e512bd6f8 100644 --- a/packages/nx/src/daemon/server/outputs-tracking.spec.ts +++ b/packages/nx/src/daemon/server/outputs-tracking.spec.ts @@ -1,3 +1,4 @@ +import { EventType } from '../../native'; import { _outputsHashesMatch, _recordOutputsHash, @@ -19,7 +20,7 @@ describe('outputs tracking', () => { it('should invalidate output when it is exact match', () => { _recordOutputsHash(['dist/app/app1'], '123'); processFileChangesInOutputs( - [{ path: 'dist/app/app1', type: 'update' }], + [{ path: 'dist/app/app1', type: EventType.update }], now ); expect(recordedHash('dist/app/app1')).toBeUndefined(); @@ -28,7 +29,7 @@ describe('outputs tracking', () => { it('should invalidate output when it is a child', () => { _recordOutputsHash(['dist/app/app1'], '123'); processFileChangesInOutputs( - [{ path: 'dist/app/app1/child', type: 'update' }], + [{ path: 'dist/app/app1/child', type: EventType.update }], now ); expect(recordedHash('dist/app/app1')).toBeUndefined(); @@ -36,13 +37,19 @@ describe('outputs tracking', () => { it('should invalidate output when it is a parent', () => { _recordOutputsHash(['dist/app/app1'], '123'); - processFileChangesInOutputs([{ path: 'dist/app', type: 'update' }], now); + processFileChangesInOutputs( + [{ path: 'dist/app', type: EventType.update }], + now + ); expect(recordedHash('dist/app/app1')).toBeUndefined(); }); it('should not invalidate anything when no match', () => { _recordOutputsHash(['dist/app/app1'], '123'); - processFileChangesInOutputs([{ path: 'dist/app2', type: 'update' }], now); + processFileChangesInOutputs( + [{ path: 'dist/app2', type: EventType.update }], + now + ); expect(recordedHash('dist/app/app1')).toEqual('123'); }); }); diff --git a/packages/nx/src/daemon/server/outputs-tracking.ts b/packages/nx/src/daemon/server/outputs-tracking.ts index 448ab5fabd5a2..c0a91af22e15b 100644 --- a/packages/nx/src/daemon/server/outputs-tracking.ts +++ b/packages/nx/src/daemon/server/outputs-tracking.ts @@ -1,9 +1,7 @@ -import { lstat } from 'fs-extra'; -import { dirname, join } from 'path'; -import { workspaceRoot } from '../../utils/workspace-root'; +import { dirname } from 'path'; +import { WatchEvent, getFilesForOutputs } from '../../native'; import { collapseExpandedOutputs } from '../../utils/collapse-expanded-outputs'; -import type { Event } from '@parcel/watcher'; -import { getFilesForOutputs } from '../../native'; +import { workspaceRoot } from '../../utils/workspace-root'; let disabled = false; @@ -66,7 +64,7 @@ async function normalizeOutputs(outputs: string[]) { } export function processFileChangesInOutputs( - changeEvents: Event[], + changeEvents: WatchEvent[], now: number = undefined ) { if (!now) { diff --git a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts index a6b59dd15b257..fe2036ffcf2ad 100644 --- a/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts +++ b/packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts @@ -140,37 +140,10 @@ function computeWorkspaceConfigHash( return hashArray(projectConfigurationStrings); } -/** - * Temporary work around to handle nested gitignores. The parcel file watcher doesn't handle them well, - * so we need to filter them out here. - * - * TODO(Cammisuli): remove after 16.4 - Rust watcher handles nested gitignores - */ -function filterUpdatedFiles(files: string[]) { - if (files.length === 0 || process.env.NX_NATIVE_WATCHER === 'true') { - return files; - } - - try { - const quoted = files.map((f) => '"' + f + '"'); - const ignored = execSync(`git check-ignore ${quoted.join(' ')}`, { - windowsHide: true, - }) - .toString() - .split('\n'); - return files.filter((f) => ignored.indexOf(f) === -1); - } catch (e) { - // none of the files were ignored - return files; - } -} - async function processCollectedUpdatedAndDeletedFiles() { try { performance.mark('hash-watched-changes-start'); - const updatedFiles = filterUpdatedFiles([ - ...collectedUpdatedFiles.values(), - ]); + const updatedFiles = [...collectedUpdatedFiles.values()]; const deletedFiles = [...collectedDeletedFiles.values()]; let updatedFileHashes = updateFilesInContext(updatedFiles, deletedFiles); performance.mark('hash-watched-changes-end'); diff --git a/packages/nx/src/daemon/server/server.ts b/packages/nx/src/daemon/server/server.ts index 4c1186b54ecf6..cc81918d014ed 100644 --- a/packages/nx/src/daemon/server/server.ts +++ b/packages/nx/src/daemon/server/server.ts @@ -1,65 +1,57 @@ -import { workspaceRoot } from '../../utils/workspace-root'; +import { existsSync, statSync } from 'fs'; import { createServer, Server, Socket } from 'net'; import { join } from 'path'; import { PerformanceObserver } from 'perf_hooks'; +import { hashArray } from '../../hasher/file-hasher'; +import { hashFile } from '../../native'; +import { consumeMessagesFromSocket } from '../../utils/consume-messages-from-socket'; +import { readJsonFile } from '../../utils/fileutils'; +import { PackageJson } from '../../utils/package-json'; +import { nxVersion } from '../../utils/versions'; +import { setupWorkspaceContext } from '../../utils/workspace-context'; +import { workspaceRoot } from '../../utils/workspace-root'; +import { writeDaemonJsonProcessCache } from '../cache'; import { FULL_OS_SOCKET_PATH, isWindows, killSocketOrPath, } from '../socket-utils'; +import { + registeredFileWatcherSockets, + removeRegisteredFileWatcherSocket, +} from './file-watching/file-watcher-sockets'; +import { handleHashTasks } from './handle-hash-tasks'; +import { + handleOutputsHashesMatch, + handleRecordOutputsHash, +} from './handle-outputs-tracking'; +import { handleProcessInBackground } from './handle-process-in-background'; +import { handleRequestFileData } from './handle-request-file-data'; +import { handleRequestProjectGraph } from './handle-request-project-graph'; +import { handleRequestShutdown } from './handle-request-shutdown'; import { serverLogger } from './logger'; import { - getOutputsWatcherSubscription, + disableOutputsTracking, + processFileChangesInOutputs, +} from './outputs-tracking'; +import { addUpdatedAndDeletedFiles } from './project-graph-incremental-recomputation'; +import { getOutputWatcherInstance, - getSourceWatcherSubscription, getWatcherInstance, handleServerProcessTermination, resetInactivityTimeout, respondToClient, respondWithErrorAndExit, SERVER_INACTIVITY_TIMEOUT_MS, - storeOutputsWatcherSubscription, storeOutputWatcherInstance, - storeProcessJsonSubscription, - storeSourceWatcherSubscription, storeWatcherInstance, } from './shutdown-utils'; import { convertChangeEventsToLogMessage, - subscribeToOutputsChanges, - subscribeToWorkspaceChanges, FileWatcherCallback, - subscribeToServerProcessJsonChanges, - watchWorkspace, watchOutputFiles, + watchWorkspace, } from './watcher'; -import { addUpdatedAndDeletedFiles } from './project-graph-incremental-recomputation'; -import { existsSync, statSync } from 'fs'; -import { handleRequestProjectGraph } from './handle-request-project-graph'; -import { handleProcessInBackground } from './handle-process-in-background'; -import { - handleOutputsHashesMatch, - handleRecordOutputsHash, -} from './handle-outputs-tracking'; -import { consumeMessagesFromSocket } from '../../utils/consume-messages-from-socket'; -import { - disableOutputsTracking, - processFileChangesInOutputs, -} from './outputs-tracking'; -import { handleRequestShutdown } from './handle-request-shutdown'; -import { - registeredFileWatcherSockets, - removeRegisteredFileWatcherSocket, -} from './file-watching/file-watcher-sockets'; -import { nxVersion } from '../../utils/versions'; -import { readJsonFile } from '../../utils/fileutils'; -import { PackageJson } from '../../utils/package-json'; -import { getDaemonProcessIdSync, writeDaemonJsonProcessCache } from '../cache'; -import { handleHashTasks } from './handle-hash-tasks'; -import { hashArray } from '../../hasher/file-hasher'; -import { handleRequestFileData } from './handle-request-file-data'; -import { setupWorkspaceContext } from '../../utils/workspace-context'; -import { hashFile } from '../../native'; let performanceObserver: PerformanceObserver | undefined; let workspaceWatcherError: Error | undefined; @@ -109,7 +101,6 @@ const server = createServer(async (socket) => { }); }); registerProcessTerminationListeners(); -registerProcessServerJsonTracking(); async function handleMessage(socket, data: string) { if (workspaceWatcherError) { @@ -239,23 +230,6 @@ function registerProcessTerminationListeners() { ); } -async function registerProcessServerJsonTracking() { - if (useNativeWatcher()) { - return; - } - - storeProcessJsonSubscription( - await subscribeToServerProcessJsonChanges(async () => { - if (getDaemonProcessIdSync() !== process.pid) { - await handleServerProcessTermination({ - server, - reason: 'this process is no longer the current daemon', - }); - } - }) - ); -} - let existingLockHash: string | undefined; function daemonIsOutdated(): boolean { @@ -419,46 +393,20 @@ export async function startServer(): Promise { // this triggers the storage of the lock file hash daemonIsOutdated(); - if (useNativeWatcher()) { - if (!getWatcherInstance()) { - storeWatcherInstance( - await watchWorkspace(server, handleWorkspaceChanges) - ); + if (!getWatcherInstance()) { + storeWatcherInstance( + await watchWorkspace(server, handleWorkspaceChanges) + ); - serverLogger.watcherLog( - `Subscribed to changes within: ${workspaceRoot} (native)` - ); - } - - if (!getOutputWatcherInstance()) { - storeOutputWatcherInstance( - await watchOutputFiles(handleOutputsChanges) - ); - } - } else { - if (!getSourceWatcherSubscription()) { - storeSourceWatcherSubscription( - await subscribeToWorkspaceChanges( - server, - handleWorkspaceChanges - ) - ); - serverLogger.watcherLog( - `Subscribed to changes within: ${workspaceRoot}` - ); - } + serverLogger.watcherLog( + `Subscribed to changes within: ${workspaceRoot} (native)` + ); + } - // temporary disable outputs tracking on linux - const outputsTrackingIsEnabled = process.platform != 'linux'; - if (outputsTrackingIsEnabled) { - if (!getOutputsWatcherSubscription()) { - storeOutputsWatcherSubscription( - await subscribeToOutputsChanges(handleOutputsChanges) - ); - } - } else { - disableOutputsTracking(); - } + if (!getOutputWatcherInstance()) { + storeOutputWatcherInstance( + await watchOutputFiles(handleOutputsChanges) + ); } return resolve(server); @@ -471,8 +419,3 @@ export async function startServer(): Promise { } }); } - -// TODO(cammisuli): remove with nx 16.6 (only our watcher will be supported) -function useNativeWatcher() { - return process.env.NX_NATIVE_WATCHER === 'true'; -} diff --git a/packages/nx/src/daemon/server/shutdown-utils.ts b/packages/nx/src/daemon/server/shutdown-utils.ts index 9bbc05053a65c..a60f84d4412a6 100644 --- a/packages/nx/src/daemon/server/shutdown-utils.ts +++ b/packages/nx/src/daemon/server/shutdown-utils.ts @@ -2,37 +2,11 @@ import { workspaceRoot } from '../../utils/workspace-root'; import type { Server, Socket } from 'net'; import { serverLogger } from './logger'; import { serializeResult } from '../socket-utils'; -import type { AsyncSubscription } from '@parcel/watcher'; import { deleteDaemonJsonProcessCache } from '../cache'; import type { Watcher } from '../../native'; export const SERVER_INACTIVITY_TIMEOUT_MS = 10800000 as const; // 10800000 ms = 3 hours -let sourceWatcherSubscription: AsyncSubscription | undefined; -let outputsWatcherSubscription: AsyncSubscription | undefined; - -export function getSourceWatcherSubscription() { - return sourceWatcherSubscription; -} - -export function storeSourceWatcherSubscription(s: AsyncSubscription) { - sourceWatcherSubscription = s; -} - -export function getOutputsWatcherSubscription() { - return outputsWatcherSubscription; -} - -export function storeOutputsWatcherSubscription(s: AsyncSubscription) { - outputsWatcherSubscription = s; -} - -let processJsonSubscription: AsyncSubscription | undefined; - -export function storeProcessJsonSubscription(s: AsyncSubscription) { - processJsonSubscription = s; -} - let watcherInstance: Watcher | undefined; export function storeWatcherInstance(instance: Watcher) { watcherInstance = instance; @@ -61,24 +35,6 @@ export async function handleServerProcessTermination({ try { server.close(); deleteDaemonJsonProcessCache(); - if (sourceWatcherSubscription) { - await sourceWatcherSubscription.unsubscribe(); - serverLogger.watcherLog( - `Unsubscribed from changes within: ${workspaceRoot} (sources)` - ); - } - if (outputsWatcherSubscription) { - await outputsWatcherSubscription.unsubscribe(); - serverLogger.watcherLog( - `Unsubscribed from changes within: ${workspaceRoot} (outputs)` - ); - } - if (processJsonSubscription) { - await processJsonSubscription.unsubscribe(); - serverLogger.watcherLog( - `Unsubscribed from changes within: ${workspaceRoot} (server-process.json)` - ); - } if (watcherInstance) { await watcherInstance.stop(); diff --git a/packages/nx/src/daemon/server/watcher.ts b/packages/nx/src/daemon/server/watcher.ts index 163a4e1e2fcc8..8e47c72844fb2 100644 --- a/packages/nx/src/daemon/server/watcher.ts +++ b/packages/nx/src/daemon/server/watcher.ts @@ -1,12 +1,4 @@ -/** - * In addition to its native performance, another great advantage of `@parcel/watcher` is that it will - * automatically take advantage of Facebook's watchman tool (https://facebook.github.io/watchman/) if - * the user has it installed (but not require it if they don't). - * - * See https://github.com/parcel-bundler/watcher for more details. - */ import { workspaceRoot } from '../../utils/workspace-root'; -import type { AsyncSubscription, Event } from '@parcel/watcher'; import { dirname, relative } from 'path'; import { FULL_OS_SOCKET_PATH } from '../socket-utils'; import { handleServerProcessTermination } from './shutdown-utils'; @@ -25,34 +17,9 @@ const ALWAYS_IGNORE = [...getAlwaysIgnore(workspaceRoot), FULL_OS_SOCKET_PATH]; export type FileWatcherCallback = ( err: Error | string | null, - changeEvents: Event[] | WatchEvent[] | null + changeEvents: WatchEvent[] | null ) => Promise; -export async function subscribeToOutputsChanges( - cb: FileWatcherCallback -): Promise { - const watcher = await import('@parcel/watcher'); - return await watcher.subscribe( - workspaceRoot, - (err, events) => { - if (err) { - return cb(err, null); - } else { - const workspaceRelativeEvents: Event[] = []; - for (const event of events) { - const workspaceRelativeEvent: Event = { - type: event.type, - path: normalizePath(relative(workspaceRoot, event.path)), - }; - workspaceRelativeEvents.push(workspaceRelativeEvent); - } - cb(null, workspaceRelativeEvents); - } - }, - watcherOptions([...ALWAYS_IGNORE]) - ); -} - export async function watchWorkspace(server: Server, cb: FileWatcherCallback) { const { Watcher } = await import('../../native'); @@ -115,88 +82,14 @@ export async function watchOutputFiles(cb: FileWatcherCallback) { return watcher; } -export async function subscribeToWorkspaceChanges( - server: Server, - cb: FileWatcherCallback -): Promise { - /** - * The imports and exports of @nx/workspace are somewhat messy and far reaching across the repo (and beyond), - * and so it is much safer for us to lazily load here `@parcel/watcher` so that its inclusion is not inadvertently - * executed by packages which do not have its necessary native binaries available. - */ - const watcher = await import('@parcel/watcher'); - const ignoreObj = getIgnoreObject(); - - return await watcher.subscribe( - workspaceRoot, - (err, events) => { - if (err) { - return cb(err, null); - } - - let hasIgnoreFileUpdate = false; - - // Most of our utilities (ignore, hashing etc) require unix-style workspace relative paths - const workspaceRelativeEvents: Event[] = []; - for (const event of events) { - const workspaceRelativeEvent: Event = { - type: event.type, - path: normalizePath(relative(workspaceRoot, event.path)), - }; - if ( - workspaceRelativeEvent.path.endsWith('.gitignore') || - workspaceRelativeEvent.path === '.nxignore' - ) { - hasIgnoreFileUpdate = true; - } - workspaceRelativeEvents.push(workspaceRelativeEvent); - } - - // If the ignore files themselves have changed we need to dynamically update our cached ignoreGlobs - if (hasIgnoreFileUpdate) { - handleServerProcessTermination({ - server, - reason: 'Stopping the daemon the set of ignored files changed.', - }); - } - - const nonIgnoredEvents = workspaceRelativeEvents - .filter(({ path }) => !!path) - .filter(({ path }) => !ignoreObj.ignores(path)); - - if (nonIgnoredEvents && nonIgnoredEvents.length > 0) { - cb(null, nonIgnoredEvents); - } - }, - watcherOptions(getIgnoredGlobs(workspaceRoot)) - ); -} - -// TODO: When we update @parcel/watcher to a version that handles negation globs, then this can be folded into the workspace watcher -export async function subscribeToServerProcessJsonChanges( - cb: () => void -): Promise { - const watcher = await import('@parcel/watcher'); - - return await watcher.subscribe( - dirname(serverProcessJsonPath), - (err, events) => { - for (const event of events) { - if (event.path === serverProcessJsonPath) { - cb(); - } - } - }, - watcherOptions([]) - ); -} - /** * NOTE: An event type of "create" will also apply to the case where the user has restored * an original version of a file after modifying/deleting it by using git, so we adjust * our log language accordingly. */ -export function convertChangeEventsToLogMessage(changeEvents: Event[]): string { +export function convertChangeEventsToLogMessage( + changeEvents: WatchEvent[] +): string { // If only a single file was changed, show the information inline if (changeEvents.length === 1) { const { path, type } = changeEvents[0]; @@ -234,15 +127,3 @@ export function convertChangeEventsToLogMessage(changeEvents: Event[]): string { return `${numCreatedOrRestoredFiles} file(s) created or restored, ${numModifiedFiles} file(s) modified, ${numDeletedFiles} file(s) deleted`; } - -function watcherOptions(ignore: string[]) { - const options: import('@parcel/watcher').Options = { - ignore, - }; - - if (platform() === 'win32') { - options.backend = 'windows'; - } - - return options; -}