From 071e0cd63c5c72b38529fed707e6ed7701c97d3b Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Thu, 22 Jun 2023 15:17:29 +0300 Subject: [PATCH 1/2] fs: add globSync implementation this is currently for internal use only, with a synchrnous API. --- lib/internal/fs/glob.js | 384 +++++++++++++++++++++++++++++++++ test/parallel/test-fs-glob.mjs | 309 ++++++++++++++++++++++++++ 2 files changed, 693 insertions(+) create mode 100644 lib/internal/fs/glob.js create mode 100644 test/parallel/test-fs-glob.mjs diff --git a/lib/internal/fs/glob.js b/lib/internal/fs/glob.js new file mode 100644 index 00000000000000..323ef2a25d434c --- /dev/null +++ b/lib/internal/fs/glob.js @@ -0,0 +1,384 @@ +'use strict'; +const { lstatSync, readdirSync } = require('fs'); +const { join, resolve } = require('path'); + +const { + kEmptyObject, +} = require('internal/util'); +const { + validateFunction, + validateObject, +} = require('internal/validators'); + +const { + ArrayFrom, + ArrayPrototypeAt, + ArrayPrototypeMap, + ArrayPrototypeFlatMap, + ArrayPrototypePop, + ArrayPrototypePush, + ArrayPrototypeSome, + SafeMap, + SafeSet, + StringPrototypeEndsWith, +} = primordials; + +let minimatch; +function lazyMinimatch() { + minimatch ??= require('internal/deps/minimatch/index'); + return minimatch; +} + +const isWindows = process.platform === 'win32'; +const isOSX = process.platform === 'darwin'; + +class Cache { + #cache = new SafeMap(); + #statsCache = new SafeMap(); + #readdirCache = new SafeMap(); + + statSync(path) { + const cached = this.#statsCache.get(path); + if (cached) { + return cached; + } + let val; + try { + val = lstatSync(path); + } catch { + val = null; + } + this.#statsCache.set(path, val); + return val; + } + addToStatCache(path, val) { + this.#statsCache.set(path, val); + } + readdirSync(path) { + const cached = this.#readdirCache.get(path); + if (cached) { + return cached; + } + let val; + try { + val = readdirSync(path, { __proto__: null, withFileTypes: true }); + } catch { + val = []; + } + this.#readdirCache.set(path, val); + return val; + } + add(path, pattern) { + let cache = this.#cache.get(path); + if (!cache) { + cache = new SafeSet(); + this.#cache.set(path, cache); + } + const originalSize = cache.size; + pattern.indexes.forEach((index) => cache.add(pattern.cacheKey(index))); + return cache.size !== originalSize + pattern.indexes.size; + } + seen(path, pattern, index) { + return this.#cache.get(path)?.has(pattern.cacheKey(index)); + } +} + +class Pattern { + #pattern; + #globStrings; + indexes; + symlinks; + last; + + constructor(pattern, globStrings, indexes, symlinks) { + this.#pattern = pattern; + this.#globStrings = globStrings; + this.indexes = indexes; + this.symlinks = symlinks; + this.last = pattern.length - 1; + } + + isLast(isDirectory) { + return this.indexes.has(this.last) || + (this.at(-1) === '' && isDirectory && + this.indexes.has(this.last - 1) && this.at(-2) === lazyMinimatch().GLOBSTAR); + } + isFirst() { + return this.indexes.has(0); + } + get hasSeenSymlinks() { + return ArrayPrototypeSome(ArrayFrom(this.indexes), (i) => !this.symlinks.has(i)); + } + at(index) { + return ArrayPrototypeAt(this.#pattern, index); + } + child(indexes, symlinks = new SafeSet()) { + return new Pattern(this.#pattern, this.#globStrings, indexes, symlinks); + } + test(index, path) { + if (index > this.#pattern.length) { + return false; + } + const pattern = this.#pattern[index]; + if (pattern === lazyMinimatch().GLOBSTAR) { + return true; + } + if (typeof pattern === 'string') { + return pattern === path; + } + if (typeof pattern?.test === 'function') { + return pattern.test(path); + } + return false; + } + + cacheKey(index) { + let key = ''; + for (let i = index; i < this.#globStrings.length; i++) { + key += this.#globStrings[i]; + if (i !== this.#globStrings.length - 1) { + key += '/'; + } + } + return key; + } +} + +class Glob { + #root; + #exclude; + #cache = new Cache(); + #results = []; + #queue = []; + #subpatterns = new SafeMap(); + constructor(patterns, options = kEmptyObject) { + validateObject(options, 'options'); + const { exclude, cwd } = options; + if (exclude != null) { + validateFunction(exclude, 'options.exclude'); + } + this.#root = cwd ?? '.'; + this.#exclude = exclude; + this.matchers = ArrayPrototypeMap(patterns, (pattern) => new (lazyMinimatch().Minimatch)(pattern, { + __proto__: null, + nocase: isWindows || isOSX, + windowsPathsNoEscape: true, + nonegate: true, + nocomment: true, + optimizationLevel: 2, + platform: process.platform, + nocaseMagicOnly: true, + })); + } + + globSync() { + ArrayPrototypePush(this.#queue, { + __proto__: null, + path: '.', + patterns: ArrayPrototypeFlatMap(this.matchers, (matcher) => ArrayPrototypeMap(matcher.set, + (pattern, i) => new Pattern( + pattern, + matcher.globParts[i], + new SafeSet([0]), + new SafeSet(), + ))), + }); + + while (this.#queue.length > 0) { + const item = ArrayPrototypePop(this.#queue); + for (let i = 0; i < item.patterns.length; i++) { + this.#addSubpatterns(item.path, item.patterns[i]); + } + this.#subpatterns + .forEach((patterns, path) => ArrayPrototypePush(this.#queue, { __proto__: null, path, patterns })); + this.#subpatterns.clear(); + } + return this.#results; + } + #addSubpattern(path, pattern) { + if (!this.#subpatterns.has(path)) { + this.#subpatterns.set(path, [pattern]); + } else { + ArrayPrototypePush(this.#subpatterns.get(path), pattern); + } + } + #addSubpatterns(path, pattern) { + const seen = this.#cache.add(path, pattern); + if (seen) { + return; + } + const fullpath = resolve(this.#root, path); + const stat = this.#cache.statSync(fullpath); + const last = pattern.last; + const isDirectory = stat?.isDirectory() || (stat?.isSymbolicLink() && pattern.hasSeenSymlinks); + const isLast = pattern.isLast(isDirectory); + const isFirst = pattern.isFirst(); + + if (isFirst && isWindows && typeof pattern.at(0) === 'string' && StringPrototypeEndsWith(pattern.at(0), ':')) { + // Absolute path, go to root + this.#addSubpattern(`${pattern.at(0)}\\`, pattern.child(new SafeSet([1]))); + return; + } + if (isFirst && pattern.at(0) === '') { + // Absolute path, go to root + this.#addSubpattern('/', pattern.child(new SafeSet([1]))); + return; + } + if (isFirst && pattern.at(0) === '..') { + // Start with .., go to parent + this.#addSubpattern('../', pattern.child(new SafeSet([1]))); + return; + } + if (isFirst && pattern.at(0) === '.') { + // Start with ., proceed + this.#addSubpattern('.', pattern.child(new SafeSet([1]))); + return; + } + + if (isLast && typeof pattern.at(-1) === 'string') { + // Add result if it exists + const p = pattern.at(-1); + const stat = this.#cache.statSync(join(fullpath, p)); + if (stat && (p || isDirectory)) { + ArrayPrototypePush(this.#results, join(path, p)); + } + if (pattern.indexes.size === 1 && pattern.indexes.has(last)) { + return; + } + } else if (isLast && pattern.at(-1) === lazyMinimatch().GLOBSTAR && + (path !== '.' || pattern.at(0) === '.' || (last === 0 && stat))) { + // If pattern ends with **, add to results + // if path is ".", add it only if pattern starts with "." or pattern is exactly "**" + ArrayPrototypePush(this.#results, path); + } + + if (!isDirectory) { + return; + } + + let children; + const firstPattern = pattern.indexes.size === 1 && pattern.at(pattern.indexes.values().next().value); + if (typeof firstPattern === 'string') { + const stat = this.#cache.statSync(join(fullpath, firstPattern)); + if (stat) { + stat.name = firstPattern; + children = [stat]; + } else { + children = []; + } + } else { + children = this.#cache.readdirSync(fullpath); + } + + for (let i = 0; i < children.length; i++) { + const entry = children[i]; + const entryPath = join(path, entry.name); + this.#cache.addToStatCache(join(fullpath, entry.name), entry); + + const subPatterns = new SafeSet(); + const nSymlinks = new SafeSet(); + for (const index of pattern.indexes) { + // For each child, chek potential patterns + if (this.#cache.seen(entryPath, pattern, index) || this.#cache.seen(entryPath, pattern, index + 1)) { + return; + } + const current = pattern.at(index); + const nextIndex = index + 1; + const next = pattern.at(nextIndex); + const fromSymlink = pattern.symlinks.has(index); + + if (current === lazyMinimatch().GLOBSTAR) { + if (entry.name[0] === '.' || (this.#exclude && this.#exclude(entry.name))) { + continue; + } + if (!fromSymlink && entry.isDirectory()) { + // If directory, add ** to its potential patterns + subPatterns.add(index); + } else if (!fromSymlink && index === last) { + // If ** is last, add to results + ArrayPrototypePush(this.#results, entryPath); + } + + // Any pattern after ** is also a potential pattern + // so we can already test it here + const nextMatches = pattern.test(nextIndex, entry.name); + if (nextMatches && nextIndex === last && !isLast) { + // If next pattern is the last one, add to results + ArrayPrototypePush(this.#results, entryPath); + } else if (nextMatches && entry.isDirectory()) { + // Pattern mached, meaning two patterns forward + // are also potential patterns + // e.g **/b/c when entry is a/b - add c to potential patterns + subPatterns.add(index + 2); + } + if ((nextMatches || pattern.at(0) === '.') && + (entry.isDirectory() || entry.isSymbolicLink()) && !fromSymlink) { + // If pattern after ** matches, or pattern starts with "." + // and entry is a directory or symlink, add to potential patterns + subPatterns.add(nextIndex); + } + + if (entry.isSymbolicLink()) { + nSymlinks.add(index); + } + + if (next === '..' && entry.isDirectory()) { + // In case pattern is "**/..", + // both parent and current directory should be added to the queue + // if this is the last pattern, add to results instead + const parent = join(path, '..'); + if (nextIndex < last) { + if (!this.#subpatterns.has(path) && !this.#cache.seen(path, pattern, nextIndex + 1)) { + this.#subpatterns.set(path, [pattern.child(new SafeSet([nextIndex + 1]))]); + } + if (!this.#subpatterns.has(parent) && !this.#cache.seen(parent, pattern, nextIndex + 1)) { + this.#subpatterns.set(parent, [pattern.child(new SafeSet([nextIndex + 1]))]); + } + } else { + if (!this.#cache.seen(path, pattern, nextIndex)) { + this.#cache.add(path, pattern.child(new SafeSet([nextIndex]))); + ArrayPrototypePush(this.#results, path); + } + if (!this.#cache.seen(path, pattern, nextIndex) || !this.#cache.seen(parent, pattern, nextIndex)) { + this.#cache.add(parent, pattern.child(new SafeSet([nextIndex]))); + ArrayPrototypePush(this.#results, parent); + } + } + } + } + if (typeof current === 'string') { + if (pattern.test(index, entry.name) && index !== last) { + // If current pattern matches entry name + // the next pattern is a potential pattern + subPatterns.add(nextIndex); + } else if (current === '.' && pattern.test(nextIndex, entry.name)) { + // If current pattern is ".", proceed to test next pattern + if (nextIndex === last) { + ArrayPrototypePush(this.#results, entryPath); + } else { + subPatterns.add(nextIndex + 1); + } + } + } + if (typeof current === 'object' && pattern.test(index, entry.name)) { + // If current pattern is a regex that matches entry name (e.g *.js) + // add next pattern to potential patterns, or to results if it's the last pattern + if (index === last) { + ArrayPrototypePush(this.#results, entryPath); + } else if (entry.isDirectory()) { + subPatterns.add(nextIndex); + } + } + } + if (subPatterns.size > 0) { + // If there are potential patterns, add to queue + this.#addSubpattern(entryPath, pattern.child(subPatterns, nSymlinks)); + } + } + } +} + +module.exports = { + __proto__: null, + Glob, +}; diff --git a/test/parallel/test-fs-glob.mjs b/test/parallel/test-fs-glob.mjs new file mode 100644 index 00000000000000..b1420fec272923 --- /dev/null +++ b/test/parallel/test-fs-glob.mjs @@ -0,0 +1,309 @@ +// Flags: --expose-internals +import * as common from '../common/index.mjs'; +import tmpdir from '../common/tmpdir.js'; +import { resolve, dirname, sep } from 'node:path'; +import { mkdir, writeFile, symlink } from 'node:fs/promises'; +import { test } from 'node:test'; +import assert from 'node:assert'; +import glob from 'internal/fs/glob'; + +tmpdir.refresh(); + +const fixtureDir = resolve(tmpdir.path, 'fixtures'); +const absDir = resolve(tmpdir.path, 'abs'); + +async function setup() { + await mkdir(fixtureDir, { recursive: true }); + await mkdir(absDir, { recursive: true }); + const files = [ + 'a/.abcdef/x/y/z/a', + 'a/abcdef/g/h', + 'a/abcfed/g/h', + 'a/b/c/d', + 'a/bc/e/f', + 'a/c/d/c/b', + 'a/cb/e/f', + 'a/x/.y/b', + 'a/z/.y/b', + ].map((f) => resolve(fixtureDir, f)); + + const symlinkTo = resolve(fixtureDir, 'a/symlink/a/b/c'); + const symlinkFrom = '../..'; + + for (const file of files) { + const f = resolve(fixtureDir, file); + const d = dirname(f); + await mkdir(d, { recursive: true }); + await writeFile(f, 'i like tests'); + } + + if (!common.isWindows) { + const d = dirname(symlinkTo); + await mkdir(d, { recursive: true }); + await symlink(symlinkFrom, symlinkTo, 'dir'); + } + + await Promise.all(['foo', 'bar', 'baz', 'asdf', 'quux', 'qwer', 'rewq'].map(async function(w) { + await mkdir(resolve(absDir, w), { recursive: true }); + })); +} + +await setup(); + +const patterns = { + 'a/c/d/*/b': ['a/c/d/c/b'], + 'a//c//d//*//b': ['a/c/d/c/b'], + 'a/*/d/*/b': ['a/c/d/c/b'], + 'a/*/+(c|g)/./d': ['a/b/c/d'], + 'a/**/[cg]/../[cg]': [ + 'a/abcdef/g', + 'a/abcfed/g', + 'a/b/c', + 'a/c', + 'a/c/d/c', + common.isWindows ? null : 'a/symlink/a/b/c', + ], + 'a/{b,c,d,e,f}/**/g': [], + 'a/b/**': ['a/b', 'a/b/c', 'a/b/c/d'], + './**/g': ['a/abcdef/g', 'a/abcfed/g'], + 'a/abc{fed,def}/g/h': ['a/abcdef/g/h', 'a/abcfed/g/h'], + 'a/abc{fed/g,def}/**/': ['a/abcdef', 'a/abcdef/g', 'a/abcfed/g'], + 'a/abc{fed/g,def}/**///**/': ['a/abcdef', 'a/abcdef/g', 'a/abcfed/g'], + '**/a': common.isWindows ? ['a'] : ['a', 'a/symlink/a'], + '**/a/**': [ + 'a', + 'a/abcdef', + 'a/abcdef/g', + 'a/abcdef/g/h', + 'a/abcfed', + 'a/abcfed/g', + 'a/abcfed/g/h', + 'a/b', + 'a/b/c', + 'a/b/c/d', + 'a/bc', + 'a/bc/e', + 'a/bc/e/f', + 'a/c', + 'a/c/d', + 'a/c/d/c', + 'a/c/d/c/b', + 'a/cb', + 'a/cb/e', + 'a/cb/e/f', + ...(common.isWindows ? [] : [ + 'a/symlink', + 'a/symlink/a', + 'a/symlink/a/b', + 'a/symlink/a/b/c', + ]), + 'a/x', + 'a/z', + ], + './**/a': common.isWindows ? ['a'] : ['a', 'a/symlink/a', 'a/symlink/a/b/c/a'], + './**/a/**/': [ + 'a', + 'a/abcdef', + 'a/abcdef/g', + 'a/abcfed', + 'a/abcfed/g', + 'a/b', + 'a/b/c', + 'a/bc', + 'a/bc/e', + 'a/c', + 'a/c/d', + 'a/c/d/c', + 'a/cb', + 'a/cb/e', + ...(common.isWindows ? [] : [ + 'a/symlink', + 'a/symlink/a', + 'a/symlink/a/b', + 'a/symlink/a/b/c', + 'a/symlink/a/b/c/a', + 'a/symlink/a/b/c/a/b', + 'a/symlink/a/b/c/a/b/c', + ]), + 'a/x', + 'a/z', + ], + './**/a/**': [ + 'a', + 'a/abcdef', + 'a/abcdef/g', + 'a/abcdef/g/h', + 'a/abcfed', + 'a/abcfed/g', + 'a/abcfed/g/h', + 'a/b', + 'a/b/c', + 'a/b/c/d', + 'a/bc', + 'a/bc/e', + 'a/bc/e/f', + 'a/c', + 'a/c/d', + 'a/c/d/c', + 'a/c/d/c/b', + 'a/cb', + 'a/cb/e', + 'a/cb/e/f', + ...(common.isWindows ? [] : [ + 'a/symlink', + 'a/symlink/a', + 'a/symlink/a/b', + 'a/symlink/a/b/c', + 'a/symlink/a/b/c/a', + 'a/symlink/a/b/c/a/b', + 'a/symlink/a/b/c/a/b/c', + ]), + 'a/x', + 'a/z', + ], + './**/a/**/a/**/': common.isWindows ? [] : [ + 'a/symlink/a', + 'a/symlink/a/b', + 'a/symlink/a/b/c', + 'a/symlink/a/b/c/a', + 'a/symlink/a/b/c/a/b', + 'a/symlink/a/b/c/a/b/c', + 'a/symlink/a/b/c/a/b/c/a', + 'a/symlink/a/b/c/a/b/c/a/b', + 'a/symlink/a/b/c/a/b/c/a/b/c', + ], + '+(a|b|c)/a{/,bc*}/**': [ + 'a/abcdef', + 'a/abcdef/g', + 'a/abcdef/g/h', + 'a/abcfed', + 'a/abcfed/g', + 'a/abcfed/g/h', + ], + '*/*/*/f': ['a/bc/e/f', 'a/cb/e/f'], + './**/f': ['a/bc/e/f', 'a/cb/e/f'], + 'a/symlink/a/b/c/a/b/c/a/b/c//a/b/c////a/b/c/**/b/c/**': common.isWindows ? [] : [ + 'a/symlink/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c', + 'a/symlink/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a', + 'a/symlink/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a/b', + 'a/symlink/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c/a/b/c', + ], + [`{./*/*,${absDir}/*}`]: [ + `${absDir}/asdf`, + `${absDir}/bar`, + `${absDir}/baz`, + `${absDir}/foo`, + `${absDir}/quux`, + `${absDir}/qwer`, + `${absDir}/rewq`, + 'a/abcdef', + 'a/abcfed', + 'a/b', + 'a/bc', + 'a/c', + 'a/cb', + common.isWindows ? null : 'a/symlink', + 'a/x', + 'a/z', + ], + [`{${absDir}/*,*}`]: [ + `${absDir}/asdf`, + `${absDir}/bar`, + `${absDir}/baz`, + `${absDir}/foo`, + `${absDir}/quux`, + `${absDir}/qwer`, + `${absDir}/rewq`, + 'a', + ], + 'a/!(symlink)/**': [ + 'a/abcdef', + 'a/abcdef/g', + 'a/abcdef/g/h', + 'a/abcfed', + 'a/abcfed/g', + 'a/abcfed/g/h', + 'a/b', + 'a/b/c', + 'a/b/c/d', + 'a/bc', + 'a/bc/e', + 'a/bc/e/f', + 'a/c', + 'a/c/d', + 'a/c/d/c', + 'a/c/d/c/b', + 'a/cb', + 'a/cb/e', + 'a/cb/e/f', + 'a/x', + 'a/z', + ], + 'a/symlink/a/**/*': common.isWindows ? [] : [ + 'a/symlink/a/b', + 'a/symlink/a/b/c', + 'a/symlink/a/b/c/a', + ], + 'a/!(symlink)/**/..': [ + 'a', + 'a/abcdef', + 'a/abcfed', + 'a/b', + 'a/bc', + 'a/c', + 'a/c/d', + 'a/cb', + ], + 'a/!(symlink)/**/../': [ + 'a', + 'a/abcdef', + 'a/abcfed', + 'a/b', + 'a/bc', + 'a/c', + 'a/c/d', + 'a/cb', + ], + 'a/!(symlink)/**/../*': [ + 'a/abcdef', + 'a/abcdef/g', + 'a/abcfed', + 'a/abcfed/g', + 'a/b', + 'a/b/c', + 'a/bc', + 'a/bc/e', + 'a/c', + 'a/c/d', + 'a/c/d/c', + 'a/cb', + 'a/cb/e', + common.isWindows ? null : 'a/symlink', + 'a/x', + 'a/z', + ], + 'a/!(symlink)/**/../*/*': [ + 'a/abcdef/g', + 'a/abcdef/g/h', + 'a/abcfed/g', + 'a/abcfed/g/h', + 'a/b/c', + 'a/b/c/d', + 'a/bc/e', + 'a/bc/e/f', + 'a/c/d', + 'a/c/d/c', + 'a/c/d/c/b', + 'a/cb/e', + 'a/cb/e/f', + common.isWindows ? null : 'a/symlink/a', + ], +}; + +for (const [pattern, expected] of Object.entries(patterns)) { + test(pattern, () => { + const actual = new glob.Glob([pattern], { cwd: fixtureDir }).globSync().sort(); + const normalized = expected.filter(Boolean).map((item) => item.replaceAll('/', sep)).sort(); + assert.deepStrictEqual(actual, normalized); + }); +} From 72ec305cbdc26e472c7bd4e3b43f6ac3e9a97c3c Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Thu, 22 Jun 2023 15:18:23 +0300 Subject: [PATCH 2/2] test_runner: support passing globs --- doc/api/test.md | 54 +++++------------ lib/internal/test_runner/runner.js | 76 +++++------------------- lib/internal/test_runner/utils.js | 15 ++--- test/parallel/test-runner-cli.js | 24 ++++---- test/parallel/test-runner-coverage.js | 13 ++-- test/parallel/test-runner-test-filter.js | 42 ------------- 6 files changed, 53 insertions(+), 171 deletions(-) delete mode 100644 test/parallel/test-runner-test-filter.js diff --git a/doc/api/test.md b/doc/api/test.md index f29fe5298b818d..1df555a8274894 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -327,52 +327,29 @@ The Node.js test runner can be invoked from the command line by passing the node --test ``` -By default, Node.js will recursively search the current directory for -JavaScript source files matching a specific naming convention. Matching files -are executed as test files. More information on the expected test file naming -convention and behavior can be found in the [test runner execution model][] -section. +By default Node.js will run all files matching these patterns: -Alternatively, one or more paths can be provided as the final argument(s) to -the Node.js command, as shown below. +* `**/*.test.?(c|m)js` +* `**/*-test.?(c|m)js` +* `**/*_test.?(c|m)js` +* `**/test-*.?(c|m)js` +* `**/test.?(c|m)js` +* `**/test/**/*.?(c|m)js` + +Alternatively, one or more glob patterns can be provided as the +final argument(s) to the Node.js command, as shown below. +Glob patterns follow the behavior of [`glob(7)`][]. ```bash -node --test test1.js test2.mjs custom_test_dir/ +node --test **/*.test.js **/*.spec.js ``` -In this example, the test runner will execute the files `test1.js` and -`test2.mjs`. The test runner will also recursively search the -`custom_test_dir/` directory for test files to execute. +Matching files are executed as test files. +More information on the test file execution can be found +in the [test runner execution model][] section. ### Test runner execution model -When searching for test files to execute, the test runner behaves as follows: - -* Any files explicitly provided by the user are executed. -* If the user did not explicitly specify any paths, the current working - directory is recursively searched for files as specified in the following - steps. -* `node_modules` directories are skipped unless explicitly provided by the - user. -* If a directory named `test` is encountered, the test runner will search it - recursively for all all `.js`, `.cjs`, and `.mjs` files. All of these files - are treated as test files, and do not need to match the specific naming - convention detailed below. This is to accommodate projects that place all of - their tests in a single `test` directory. -* In all other directories, `.js`, `.cjs`, and `.mjs` files matching the - following patterns are treated as test files: - * `^test$` - Files whose basename is the string `'test'`. Examples: - `test.js`, `test.cjs`, `test.mjs`. - * `^test-.+` - Files whose basename starts with the string `'test-'` - followed by one or more characters. Examples: `test-example.js`, - `test-another-example.mjs`. - * `.+[\.\-\_]test$` - Files whose basename ends with `.test`, `-test`, or - `_test`, preceded by one or more characters. Examples: `example.test.js`, - `example-test.cjs`, `example_test.mjs`. - * Other file types understood by Node.js such as `.node` and `.json` are not - automatically executed by the test runner, but are supported if explicitly - provided on the command line. - Each matching test file is executed in a separate child process. If the child process finishes with an exit code of 0, the test is considered passing. Otherwise, the test is considered to be a failure. Test files must be @@ -2459,6 +2436,7 @@ added: [`context.skip`]: #contextskipmessage [`context.todo`]: #contexttodomessage [`describe()`]: #describename-options-fn +[`glob(7)`]: https://man7.org/linux/man-pages/man7/glob.7.html [`run()`]: #runoptions [`test()`]: #testname-options-fn [describe options]: #describename-options-fn diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 09d0df2e660e71..af121273c9e638 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -1,11 +1,12 @@ 'use strict'; const { - ArrayFrom, ArrayIsArray, + ArrayPrototypeEvery, ArrayPrototypeFilter, ArrayPrototypeForEach, ArrayPrototypeIncludes, ArrayPrototypeMap, + ArrayPrototypeJoin, ArrayPrototypePush, ArrayPrototypeShift, ArrayPrototypeSlice, @@ -27,7 +28,6 @@ const { } = primordials; const { spawn } = require('child_process'); -const { readdirSync, statSync } = require('fs'); const { finished } = require('internal/streams/end-of-stream'); const { DefaultDeserializer, DefaultSerializer } = require('v8'); // TODO(aduh95): switch to internal/readline/interface when backporting to Node.js 16.x is no longer a concern. @@ -60,10 +60,9 @@ const { const { convertStringToRegExp, countCompletedTest, - doesPathMatchFilter, - isSupportedFileType, + kDefaultPattern, } = require('internal/test_runner/utils'); -const { basename, join, resolve } = require('path'); +const { Glob } = require('internal/fs/glob'); const { once } = require('events'); const { triggerUncaughtException, @@ -79,66 +78,23 @@ const kCanceledTests = new SafeSet() let kResistStopPropagation; -// TODO(cjihrig): Replace this with recursive readdir once it lands. -function processPath(path, testFiles, options) { - const stats = statSync(path); - - if (stats.isFile()) { - if (options.userSupplied || - (options.underTestDir && isSupportedFileType(path)) || - doesPathMatchFilter(path)) { - testFiles.add(path); - } - } else if (stats.isDirectory()) { - const name = basename(path); - - if (!options.userSupplied && name === 'node_modules') { - return; - } - - // 'test' directories get special treatment. Recursively add all .js, - // .cjs, and .mjs files in the 'test' directory. - const isTestDir = name === 'test'; - const { underTestDir } = options; - const entries = readdirSync(path); - - if (isTestDir) { - options.underTestDir = true; - } - - options.userSupplied = false; - - for (let i = 0; i < entries.length; i++) { - processPath(join(path, entries[i]), testFiles, options); - } - - options.underTestDir = underTestDir; - } -} - function createTestFileList() { const cwd = process.cwd(); - const hasUserSuppliedPaths = process.argv.length > 1; - const testPaths = hasUserSuppliedPaths ? - ArrayPrototypeSlice(process.argv, 1) : [cwd]; - const testFiles = new SafeSet(); - - try { - for (let i = 0; i < testPaths.length; i++) { - const absolutePath = resolve(testPaths[i]); - - processPath(absolutePath, testFiles, { userSupplied: true }); - } - } catch (err) { - if (err?.code === 'ENOENT') { - console.error(`Could not find '${err.path}'`); - process.exit(kGenericUserError); - } + const hasUserSuppliedPattern = process.argv.length > 1; + const patterns = hasUserSuppliedPattern ? ArrayPrototypeSlice(process.argv, 1) : [kDefaultPattern]; + const glob = new Glob(patterns, { + __proto__: null, + cwd, + exclude: (name) => name === 'node_modules', + }); + const results = glob.globSync(); - throw err; + if (hasUserSuppliedPattern && results.length === 0 && ArrayPrototypeEvery(glob.matchers, (m) => !m.hasMagic())) { + console.error(`Could not find '${ArrayPrototypeJoin(patterns, ', ')}'`); + process.exit(kGenericUserError); } - return ArrayPrototypeSort(ArrayFrom(testFiles)); + return ArrayPrototypeSort(results); } function filterExecArgv(arg, i, arr) { diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index da429b5421a45a..69b59b25410ff6 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -19,7 +19,7 @@ const { StringPrototypeSlice, } = primordials; -const { basename, relative } = require('path'); +const { relative } = require('path'); const { createWriteStream } = require('fs'); const { pathToFileURL } = require('internal/url'); const { createDeferredPromise } = require('internal/util'); @@ -44,16 +44,10 @@ const coverageColors = { const kMultipleCallbackInvocations = 'multipleCallbackInvocations'; const kRegExpPattern = /^\/(.*)\/([a-z]*)$/; -const kSupportedFileExtensions = /\.[cm]?js$/; -const kTestFilePattern = /((^test(-.+)?)|(.+[.\-_]test))\.[cm]?js$/; -function doesPathMatchFilter(p) { - return RegExpPrototypeExec(kTestFilePattern, basename(p)) !== null; -} +const kPatterns = ['test', 'test/**/*', 'test-*', '*[.-_]test']; +const kDefaultPattern = `**/{${ArrayPrototypeJoin(kPatterns, ',')}}.?(c|m)js`; -function isSupportedFileType(p) { - return RegExpPrototypeExec(kSupportedFileExtensions, p) !== null; -} function createDeferredCallback() { let calledCount = 0; @@ -414,9 +408,8 @@ module.exports = { convertStringToRegExp, countCompletedTest, createDeferredCallback, - doesPathMatchFilter, - isSupportedFileType, isTestFailureError, + kDefaultPattern, parseCommandLine, setupTestReporters, getCoverageReport, diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js index 1af875e7e24def..704e72b2df49d6 100644 --- a/test/parallel/test-runner-cli.js +++ b/test/parallel/test-runner-cli.js @@ -21,8 +21,8 @@ const testFixtures = fixtures.path('test-runner'); { // Default behavior. node_modules is ignored. Files that don't match the // pattern are ignored except in test/ directories. - const args = ['--test', testFixtures]; - const child = spawnSync(process.execPath, args); + const args = ['--test']; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); @@ -30,19 +30,19 @@ const testFixtures = fixtures.path('test-runner'); const stdout = child.stdout.toString(); assert.match(stdout, /ok 1 - this should pass/); assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); assert.match(stdout, /ok 4 - this should pass/); } { // Same but with a prototype mutation in require scripts. - const args = ['--require', join(testFixtures, 'protoMutation.js'), '--test', testFixtures]; - const child = spawnSync(process.execPath, args); + const args = ['--require', join(testFixtures, 'protoMutation.js'), '--test']; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); const stdout = child.stdout.toString(); assert.match(stdout, /ok 1 - this should pass/); assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); assert.match(stdout, /ok 4 - this should pass/); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); @@ -51,23 +51,19 @@ const testFixtures = fixtures.path('test-runner'); { // User specified files that don't match the pattern are still run. - const args = ['--test', testFixtures, join(testFixtures, 'index.js')]; - const child = spawnSync(process.execPath, args); + const args = ['--test', join(testFixtures, 'index.js')]; + const child = spawnSync(process.execPath, args, { cwd: testFixtures }); assert.strictEqual(child.status, 1); assert.strictEqual(child.signal, null); assert.strictEqual(child.stderr.toString(), ''); const stdout = child.stdout.toString(); assert.match(stdout, /not ok 1 - .+index\.js/); - assert.match(stdout, /ok 2 - this should pass/); - assert.match(stdout, /not ok 3 - this should fail/); - assert.match(stdout, /ok 4 - .+subdir.+subdir_test\.js/); - assert.match(stdout, /ok 5 - this should pass/); } { // Searches node_modules if specified. - const args = ['--test', join(testFixtures, 'node_modules')]; + const args = ['--test', join(testFixtures, 'node_modules/*.js')]; const child = spawnSync(process.execPath, args); assert.strictEqual(child.status, 1); @@ -89,7 +85,7 @@ const testFixtures = fixtures.path('test-runner'); const stdout = child.stdout.toString(); assert.match(stdout, /ok 1 - this should pass/); assert.match(stdout, /not ok 2 - this should fail/); - assert.match(stdout, /ok 3 - .+subdir.+subdir_test\.js/); + assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/); assert.match(stdout, /ok 4 - this should pass/); } diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index dcce8c1730ea84..9377f1bb509328 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -153,13 +153,13 @@ test('coverage is combined for multiple processes', skipIfNoInspector, () => { let report = [ '# start of coverage report', '# file | line % | branch % | funcs % | uncovered lines', - '# test/fixtures/v8-coverage/combined_coverage/common.js | 89.86 | ' + + '# common.js | 89.86 | ' + '62.50 | 100.00 | 8, 13, 14, 18, 34, 35, 53', - '# test/fixtures/v8-coverage/combined_coverage/first.test.js | 83.33 | ' + + '# first.test.js | 83.33 | ' + '100.00 | 50.00 | 5, 6', - '# test/fixtures/v8-coverage/combined_coverage/second.test.js | 100.00 ' + + '# second.test.js | 100.00 ' + '| 100.00 | 100.00 | ', - '# test/fixtures/v8-coverage/combined_coverage/third.test.js | 100.00 | ' + + '# third.test.js | 100.00 | ' + '100.00 | 100.00 | ', '# all files | 92.11 | 72.73 | 88.89 |', '# end of coverage report', @@ -171,10 +171,11 @@ test('coverage is combined for multiple processes', skipIfNoInspector, () => { const fixture = fixtures.path('v8-coverage', 'combined_coverage'); const args = [ - '--test', '--experimental-test-coverage', '--test-reporter', 'tap', fixture, + '--test', '--experimental-test-coverage', '--test-reporter', 'tap', ]; const result = spawnSync(process.execPath, args, { - env: { ...process.env, NODE_TEST_TMPDIR: tmpdir.path } + env: { ...process.env, NODE_TEST_TMPDIR: tmpdir.path }, + cwd: fixture, }); assert.strictEqual(result.stderr.toString(), ''); diff --git a/test/parallel/test-runner-test-filter.js b/test/parallel/test-runner-test-filter.js deleted file mode 100644 index b6afba22a2e814..00000000000000 --- a/test/parallel/test-runner-test-filter.js +++ /dev/null @@ -1,42 +0,0 @@ -// Flags: --expose-internals -'use strict'; -require('../common'); -const assert = require('assert'); -const { doesPathMatchFilter } = require('internal/test_runner/utils'); - -// Paths expected to match -[ - 'test.js', - 'test.cjs', - 'test.mjs', - 'test-foo.js', - 'test-foo.cjs', - 'test-foo.mjs', - 'foo.test.js', - 'foo.test.cjs', - 'foo.test.mjs', - 'foo-test.js', - 'foo-test.cjs', - 'foo-test.mjs', - 'foo_test.js', - 'foo_test.cjs', - 'foo_test.mjs', -].forEach((p) => { - assert.strictEqual(doesPathMatchFilter(p), true); -}); - -// Paths expected not to match -[ - 'test', - 'test.djs', - 'test.cs', - 'test.mj', - 'foo.js', - 'test-foo.sj', - 'test.foo.js', - 'test_foo.js', - 'testfoo.js', - 'foo-test1.mjs', -].forEach((p) => { - assert.strictEqual(doesPathMatchFilter(p), false); -});