diff --git a/bun.lockb b/bun.lockb index 7a921a126..cd01e4f32 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/packages/knip/package.json b/packages/knip/package.json index bfce4ae35..857ebf4b4 100644 --- a/packages/knip/package.json +++ b/packages/knip/package.json @@ -61,7 +61,6 @@ "@snyk/github-codeowners": "1.1.0", "easy-table": "1.2.0", "fast-glob": "^3.3.2", - "file-entry-cache": "8.0.0", "jiti": "^1.21.0", "js-yaml": "^4.1.0", "minimist": "^1.2.8", @@ -84,7 +83,6 @@ "@jest/types": "^29.6.3", "@release-it/bumper": "^6.0.1", "@types/bun": "^1.1.4", - "@types/file-entry-cache": "5.0.4", "@types/js-yaml": "^4.0.9", "@types/minimist": "^1.2.5", "@types/picomatch": "2.3.3", diff --git a/packages/knip/src/CacheConsultant.ts b/packages/knip/src/CacheConsultant.ts index 980eccd03..33d3f425f 100644 --- a/packages/knip/src/CacheConsultant.ts +++ b/packages/knip/src/CacheConsultant.ts @@ -1,6 +1,6 @@ -import fileEntryCache, { type FileEntryCache, type FileDescriptor } from 'file-entry-cache'; import { timerify } from './util/Performance.js'; import parsedArgValues from './util/cli-arguments.js'; +import { type FileDescriptor, FileEntryCache } from './util/file-entry-cache.js'; import { cwd, join } from './util/path.js'; import { version } from './version.js'; @@ -10,27 +10,17 @@ const { cache: isCache = false, watch: isWatch = false } = parsedArgValues; const cacheLocation = parsedArgValues['cache-location'] ?? defaultCacheLocation; -interface FD extends FileDescriptor { - readonly meta?: { - readonly size?: number; - readonly mtime?: number; - readonly hash?: string; - data?: T; - }; -} - -const create = timerify(fileEntryCache.create, 'createCache'); - -const dummyFileDescriptor = { key: '', changed: true, notFound: true, meta: undefined }; +// biome-ignore lint/suspicious/noExplicitAny: deal with it +const dummyFileDescriptor: FileDescriptor = { key: '', changed: true, notFound: true }; const isEnabled = isCache || isWatch; export class CacheConsultant { - private cache: undefined | FileEntryCache; + private cache: undefined | FileEntryCache; constructor(name: string) { if (isCache) { const cacheName = `${name.replace(/[^a-z0-9]/g, '-').replace(/-*$/, '')}-${version}`; - this.cache = create(cacheName, cacheLocation); + this.cache = new FileEntryCache(cacheName, cacheLocation); this.reconcile = timerify(this.cache.reconcile).bind(this.cache); this.getFileDescriptor = timerify(this.cache.getFileDescriptor).bind(this.cache); } @@ -38,7 +28,7 @@ export class CacheConsultant { static getCacheLocation() { return cacheLocation; } - public getFileDescriptor(file: string): FD { + public getFileDescriptor(file: string): FileDescriptor { if (isEnabled && this.cache) return this.cache?.getFileDescriptor(file); return dummyFileDescriptor; } diff --git a/packages/knip/src/ProjectPrincipal.ts b/packages/knip/src/ProjectPrincipal.ts index 16d851a25..551e1493a 100644 --- a/packages/knip/src/ProjectPrincipal.ts +++ b/packages/knip/src/ProjectPrincipal.ts @@ -15,7 +15,6 @@ import { timerify } from './util/Performance.js'; import { compact } from './util/array.js'; import { getPackageNameFromModuleSpecifier, isStartsLikePackageName, sanitizeSpecifier } from './util/modules.js'; import { dirname, extname, isInNodeModules, join } from './util/path.js'; -import { _deserialize, _serialize } from './util/serialize.js'; import type { ToSourceFilePath } from './util/to-source-path.js'; // These compiler options override local options @@ -226,7 +225,7 @@ export class ProjectPrincipal { getPrincipalByFilePath: (filePath: string) => undefined | ProjectPrincipal ) { const fd = this.cache.getFileDescriptor(filePath); - if (!fd.changed && fd.meta?.data) return _deserialize(fd.meta.data); + if (!fd.changed && fd.meta?.data) return fd.meta.data; const typeChecker = this.backend.typeChecker; @@ -339,7 +338,8 @@ export class ProjectPrincipal { for (const [filePath, file] of graph.entries()) { const fd = this.cache.getFileDescriptor(filePath); if (!fd?.meta) continue; - fd.meta.data = _serialize(file); + const { imported, internalImportCache, ...clone } = file; + fd.meta.data = clone; } this.cache.reconcile(); } diff --git a/packages/knip/src/WorkspaceWorker.ts b/packages/knip/src/WorkspaceWorker.ts index 0f93d2bba..0ed56c262 100644 --- a/packages/knip/src/WorkspaceWorker.ts +++ b/packages/knip/src/WorkspaceWorker.ts @@ -34,6 +34,8 @@ type WorkspaceManagerOptions = { export type ReferencedDependencies = Set<[string, string]>; +type CacheItem = { resolveEntryPaths?: string[]; resolveConfig?: string[] }; + const nullConfig: EnsuredPluginConfiguration = { config: null, entry: null, project: null }; const initEnabledPluginsMap = () => @@ -65,7 +67,7 @@ export class WorkspaceWorker { enabledPlugins: PluginName[] = []; enabledPluginsInAncestors: string[]; - cache: CacheConsultant; + cache: CacheConsultant; constructor({ name, @@ -286,21 +288,28 @@ export class WorkspaceWorker { if (hasResolveEntryPaths || shouldRunConfigResolver) { const isManifest = basename(configFilePath) === 'package.json'; const fd = isManifest ? undefined : this.cache.getFileDescriptor(configFilePath); - const config = - fd?.meta?.data && !fd.changed - ? fd.meta.data - : await loadConfigForPlugin(configFilePath, plugin, opts, pluginName); - if (config) { - if (hasResolveEntryPaths) { - const dependencies = (await plugin.resolveEntryPaths?.(config, opts)) ?? []; - for (const id of dependencies) configEntryPaths.push(id); - } - if (shouldRunConfigResolver) { - const dependencies = (await plugin.resolveConfig?.(config, opts)) ?? []; - for (const id of dependencies) addDependency(id, configFilePath); - } - if (!isManifest && fd?.changed && fd.meta) fd.meta.data = config; + if (fd?.meta?.data && !fd.changed) { + if (fd.meta.data.resolveEntryPaths) + for (const id of fd.meta.data.resolveEntryPaths) configEntryPaths.push(id); + if (fd.meta.data.resolveConfig) + for (const id of fd.meta.data.resolveConfig) addDependency(id, configFilePath); + } else { + const config = await loadConfigForPlugin(configFilePath, plugin, opts, pluginName); + const data: CacheItem = {}; + if (config) { + if (hasResolveEntryPaths) { + const dependencies = (await plugin.resolveEntryPaths?.(config, opts)) ?? []; + for (const id of dependencies) configEntryPaths.push(id); + data.resolveEntryPaths = dependencies; + } + if (shouldRunConfigResolver) { + const dependencies = (await plugin.resolveConfig?.(config, opts)) ?? []; + for (const id of dependencies) addDependency(id, configFilePath); + data.resolveConfig = dependencies; + } + if (!isManifest && fd?.changed && fd.meta) fd.meta.data = data; + } } } } diff --git a/packages/knip/src/util/file-entry-cache.ts b/packages/knip/src/util/file-entry-cache.ts new file mode 100644 index 000000000..75412afc4 --- /dev/null +++ b/packages/knip/src/util/file-entry-cache.ts @@ -0,0 +1,130 @@ +import fs from 'node:fs'; +import { timerify } from './Performance.js'; +import { debugLog } from './debug.js'; +import { isDirectory, isFile } from './fs.js'; +import { dirname, isAbsolute, resolve } from './path.js'; +import { deserialize, serialize } from './serialize.js'; + +type MetaData = { size: number; mtime: number; data?: T }; + +export type FileDescriptor = { + key: string; + changed?: boolean; + notFound?: boolean; + err?: unknown; + meta?: MetaData; +}; + +const cwd = process.cwd(); + +const createCache = (filePath: string) => { + try { + return deserialize(fs.readFileSync(filePath)); + } catch (_err) { + debugLog('*', `Error reading cache from ${filePath}`); + } +}; + +const create = timerify(createCache); + +export class FileEntryCache { + filePath: string; + cache = new Map>(); + normalizedEntries = new Map>(); + + constructor(cacheId: string, _path: string) { + this.filePath = isAbsolute(_path) ? resolve(_path, cacheId) : resolve(cwd, _path, cacheId); + if (isFile(this.filePath)) this.cache = create(this.filePath); + this.removeNotFoundFiles(); + } + + removeNotFoundFiles() { + for (const filePath of this.normalizedEntries.keys()) { + try { + fs.statSync(filePath); + } catch (error) { + // @ts-expect-error + if (error.code === 'ENOENT') this.cache.delete(filePath); + } + } + } + + getFileDescriptor(filePath: string): FileDescriptor { + let fstat: fs.Stats; + + try { + if (!isAbsolute(filePath)) filePath = resolve(filePath); + fstat = fs.statSync(filePath); + } catch (error: unknown) { + this.removeEntry(filePath); + return { key: filePath, notFound: true, err: error }; + } + + return this._getFileDescriptorUsingMtimeAndSize(filePath, fstat); + } + + _getFileDescriptorUsingMtimeAndSize(filePath: string, fstat: fs.Stats) { + let meta = this.cache.get(filePath); + const cacheExists = Boolean(meta); + + const cSize = fstat.size; + const cTime = fstat.mtime.getTime(); + + let isDifferentDate: undefined | boolean; + let isDifferentSize: undefined | boolean; + + if (meta) { + isDifferentDate = cTime !== meta.mtime; + isDifferentSize = cSize !== meta.size; + } else { + meta = { size: cSize, mtime: cTime }; + } + + const fd: FileDescriptor = { + key: filePath, + changed: !cacheExists || isDifferentDate || isDifferentSize, + meta, + }; + + this.normalizedEntries.set(filePath, fd); + + return fd; + } + + removeEntry(entryName: string) { + if (!isAbsolute(entryName)) entryName = resolve(cwd, entryName); + this.normalizedEntries.delete(entryName); + this.cache.delete(entryName); + } + + _getMetaForFileUsingMtimeAndSize(cacheEntry: FileDescriptor) { + const stat = fs.statSync(cacheEntry.key); + const meta = Object.assign(cacheEntry.meta ?? {}, { + size: stat.size, + mtime: stat.mtime.getTime(), + }); + return meta; + } + + reconcile() { + this.removeNotFoundFiles(); + + for (const [entryName, cacheEntry] of this.normalizedEntries.entries()) { + try { + const meta = this._getMetaForFileUsingMtimeAndSize(cacheEntry); + this.cache.set(entryName, meta); + } catch (error) { + // @ts-expect-error + if (error.code !== 'ENOENT') throw error; + } + } + + try { + const dir = dirname(this.filePath); + if (!isDirectory(dir)) fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(this.filePath, serialize(this.cache)); + } catch (_err) { + debugLog('*', `Error writing cache to ${this.filePath}`); + } + } +} diff --git a/packages/knip/src/util/serialize.ts b/packages/knip/src/util/serialize.ts index 091d3845c..c15595dea 100644 --- a/packages/knip/src/util/serialize.ts +++ b/packages/knip/src/util/serialize.ts @@ -1,41 +1,14 @@ -import type { FileNode } from '../types/dependency-graph.js'; -import { timerify } from './Performance.js'; - -// biome-ignore lint/suspicious/noExplicitAny: deal with it -const serializeObj = (obj: any): any => { - if (!obj) return obj; - if (obj instanceof Set) return Array.from(obj); - if (obj instanceof Map) { - const o: { [key: string]: unknown } = { _m: 1 }; - for (const [key, value] of obj) o[key] = serializeObj(value); - return o; - } - if (typeof obj === 'object') for (const key in obj) obj[key] = serializeObj(obj[key]); - return obj; -}; - -// biome-ignore lint/suspicious/noExplicitAny: deal with it -const deserializeObj = (obj: any): any => { - if (!obj) return obj; - if (Array.isArray(obj)) return new Set(obj); - if (obj._m) { - const map = new Map(); - for (const key in obj) key !== '_m' && map.set(key, deserializeObj(obj[key])); - return map; - } - if (typeof obj === 'object') for (const key in obj) obj[key] = deserializeObj(obj[key]); - return obj; -}; - -const serialize = (data: FileNode): FileNode => { - const clone = structuredClone(data); - clone.imported = undefined; - clone.internalImportCache = undefined; - return serializeObj(clone); -}; - -const deserialize = (data: FileNode): FileNode => deserializeObj(data); - -export const _serialize = timerify(serialize); - -export const _deserialize = timerify(deserialize); +// biome-ignore lint: well +let s: (data: any) => Buffer, d: (buffer: Buffer) => any; + +if (typeof Bun !== 'undefined') { + const { serialize, deserialize } = await import('bun:jsc'); + s = serialize; + d = deserialize; +} else { + const { serialize, deserialize } = await import('node:v8'); + s = serialize; + d = deserialize; +} + +export { s as serialize, d as deserialize }; diff --git a/packages/knip/test/util/serialize.test.ts b/packages/knip/test/util/serialize.test.ts index 675aeda2e..5a4370cc8 100644 --- a/packages/knip/test/util/serialize.test.ts +++ b/packages/knip/test/util/serialize.test.ts @@ -1,6 +1,6 @@ import { test } from 'bun:test'; import assert from 'node:assert/strict'; -import { _deserialize, _serialize } from '../../src/util/serialize.js'; +import { deserialize, serialize } from '../../src/util/serialize.js'; test('Should serialize and deserialize file back to original', () => { const file = { @@ -63,5 +63,5 @@ test('Should serialize and deserialize file back to original', () => { traceRefs: new Set(['ref']), }; - assert.deepEqual(_deserialize(_serialize(file)), file); + assert.deepEqual(deserialize(serialize(file)), file); });