From a8c98ab00a2b1bc71be9d5322e68469970d36cd5 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Fri, 21 Apr 2017 13:48:48 -0700 Subject: [PATCH 1/9] fix bug: ignore with glob pattern (closes #1), add filter option --- klaw-sync.js | 84 +++++++------- test/test.js | 311 +++++++++++++++++++++++++++++---------------------- 2 files changed, 217 insertions(+), 178 deletions(-) diff --git a/klaw-sync.js b/klaw-sync.js index b41b7bf..1bf5825 100644 --- a/klaw-sync.js +++ b/klaw-sync.js @@ -1,60 +1,54 @@ 'use strict' -var path = require('path') -var mm = require('micromatch') -var fs +const path = require('path') +const mm = require('micromatch') +let fs try { fs = require('graceful-fs') } catch (e) { fs = require('fs') } -function _procPath (dir, pathItem, opts, list) { - var nestedPath - var stat - // here since dir already resolved, we use string concatenation - // which showed faster performance than path.join() and path.resolve() - if (path.sep === '/') { - nestedPath = dir + '/' + pathItem - } else { - nestedPath = dir + '\\' + pathItem - } - stat = fs.lstatSync(nestedPath) - if (stat.isDirectory()) { - if (!opts.nodir) { - list.push({path: nestedPath, stats: stat}) - } - list = walkSync(nestedPath, opts, list) - } else { - if (!opts.nofile) { - list.push({path: nestedPath, stats: stat}) +function klawSync (dir, opts, ls) { + function procPath (pathItem) { + const stat = fs.lstatSync(pathItem) + const item = {path: pathItem, stats: stat} + if (stat.isDirectory()) { + if (!opts.nodir) { + if (opts.ignore) { + if (mm(pathItem, opts.ignore).length === 0) ls.push(item) + } else if (opts.filter) { + if (opts.filter(item)) ls.push(item) + } else { + ls.push(item) + } + } + ls = klawSync(pathItem, opts, ls) + } else { + if (!opts.nofile) { + if (opts.ignore) { + if (mm(pathItem, opts.ignore).length === 0) ls.push(item) + } else if (opts.filter) { + if (opts.filter(item)) ls.push(item) + } else { + ls.push(item) + } + } } } -} -function walkSync (dir, opts, list) { - var files - var ignore = [] opts = opts || {} - list = list || [] + ls = ls || [] dir = path.resolve(dir) - try { - files = fs.readdirSync(dir) - if (opts.ignore) { - ignore = mm(files, opts.ignore) - } - } catch (er) { - throw er - } - - for (var i = 0; i < files.length; i += 1) { - var file = files[i] - if (ignore.length > 0) { - if (ignore.indexOf(file) === -1) _procPath(dir, file, opts, list) - } else { - _procPath(dir, file, opts, list) - } + const files = fs.readdirSync(dir) + for (let i = 0; i < files.length; i += 1) { + // here dir already resolved, we use string concatenation since + // showed better performance than path.join() and path.resolve() + let pathItem + if (path.sep === '/') pathItem = dir + '/' + files[i] + else pathItem = dir + '\\' + files[i] + procPath(pathItem) } - return list + return ls } -module.exports = walkSync +module.exports = klawSync diff --git a/test/test.js b/test/test.js index 211f1ac..a23a912 100644 --- a/test/test.js +++ b/test/test.js @@ -1,191 +1,236 @@ 'use strict' -var assert = require('assert') -var os = require('os') -var path = require('path') -var fs = require('fs-extra') +const assert = require('assert') +const os = require('os') +const path = require('path') +const fs = require('fs-extra') +const klawSync = require('../klaw-sync.js') -var klawSync = require('../klaw-sync.js') +describe('klaw-sync', () => { + const TEST_DIR = path.join(os.tmpdir(), 'klaw-sync') + const dirnames = ['dir1', 'dir2', 'dir2/dir2_1', 'dir2/dir2_1/dir2_1_1'] + const filenames = ['dir1/file1_2', 'dir2/dir2_1/file2_1_1', 'file1'] + let DIRS + let FILES -describe('klaw-sync', function () { - var TEST_DIR - var FIXTURES_DIR - var DIRS - var FILES - var dirnames = ['dir1', 'dir2', 'dir2/dir2_1', 'dir2/dir2_1/dir2_1_1'] - var filenames = ['dir1/file1_2', 'dir2/dir2_1/file2_1_1', 'file1'] - - beforeEach(function (done) { - TEST_DIR = path.join(os.tmpdir(), 'klaw-sync') - FIXTURES_DIR = path.join(TEST_DIR, 'fixtures') + beforeEach(() => { fs.emptyDirSync(TEST_DIR) - DIRS = dirnames.map(function (dir) { - return path.join(FIXTURES_DIR, dir) - }) - FILES = filenames.map(function (f) { - return path.join(FIXTURES_DIR, f) - }) - DIRS.forEach(function (dir) { - fs.ensureDirSync(dir) - }) - FILES.forEach(function (f) { - fs.ensureFileSync(f) - }) - done() - }) - afterEach(function (done) { - fs.removeSync(TEST_DIR) - done() + DIRS = dirnames.map(dir => path.join(TEST_DIR, dir)) + FILES = filenames.map(f => path.join(TEST_DIR, f)) + DIRS.forEach(dir => fs.ensureDirSync(dir)) + FILES.forEach(f => fs.ensureFileSync(f)) }) - it('should return an error if the source dir does not exist', function (done) { + afterEach(() => fs.removeSync(TEST_DIR)) + + it('should return an error if the source dir does not exist', () => { try { klawSync('dirDoesNotExist/') } catch (err) { assert.equal(err.code, 'ENOENT') - done() } }) - it('should return an error if the source is not a dir', function (done) { + it('should return an error if the source is not a dir', () => { try { klawSync(FILES[0]) } catch (err) { assert.equal(err.code, 'ENOTDIR') - done() } }) - it('should return all items of a dir containing path and stats object', function (done) { - var expectedItems = [ - {path: DIRS[0], stats: fs.lstatSync(DIRS[0])}, - {path: FILES[0], stats: fs.lstatSync(FILES[0])}, - {path: DIRS[1], stats: fs.lstatSync(DIRS[1])}, - {path: DIRS[2], stats: fs.lstatSync(DIRS[2])}, - {path: DIRS[3], stats: fs.lstatSync(DIRS[3])}, - {path: FILES[1], stats: fs.lstatSync(FILES[1])}, - {path: FILES[2], stats: fs.lstatSync(FILES[2])} + it('should return all items of a dir containing path and stats object', () => { + const paths = [ + {path: DIRS[0], stats: fs.statSync(DIRS[0])}, + {path: FILES[0], stats: fs.statSync(FILES[0])}, + {path: DIRS[1], stats: fs.statSync(DIRS[1])}, + {path: DIRS[2], stats: fs.statSync(DIRS[2])}, + {path: DIRS[3], stats: fs.statSync(DIRS[3])}, + {path: FILES[1], stats: fs.statSync(FILES[1])}, + {path: FILES[2], stats: fs.statSync(FILES[2])} ] - var items = klawSync(FIXTURES_DIR) - assert.equal(items.length, expectedItems.length) - items.forEach(function (elem, i) { - assert.deepEqual(elem, expectedItems[i]) - assert.strictEqual(elem.path, expectedItems[i].path) - assert.deepEqual(elem.stats, expectedItems[i].stats) + const items = klawSync(TEST_DIR) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) }) - done() }) - it('should return only files if opts.nodir is true', function (done) { - var expectedItems = [ + it('should return only files if opts.nodir is true', () => { + const filesOnly = [ {path: FILES[0], stats: fs.lstatSync(FILES[0])}, {path: FILES[1], stats: fs.lstatSync(FILES[1])}, {path: FILES[2], stats: fs.lstatSync(FILES[2])} ] - var actualFiles = klawSync(FIXTURES_DIR, {nodir: true}) - assert.equal(actualFiles.length, expectedItems.length) - actualFiles.forEach(function (elem, i) { - assert.deepEqual(elem, expectedItems[i]) - assert.strictEqual(elem.path, expectedItems[i].path) - assert.deepEqual(elem.stats, expectedItems[i].stats) + const files = klawSync(TEST_DIR, {nodir: true}) + assert.equal(files.length, filesOnly.length) + files.forEach((f, i) => { + assert.deepEqual(f, filesOnly[i]) + assert.strictEqual(f.path, filesOnly[i].path) + assert.deepEqual(f.stats, filesOnly[i].stats) }) - done() }) - it('should return only dirs if opts.nofile is true', function (done) { - var expectedItems = [ + it('should return only dirs if opts.nofile is true', () => { + const dirsOnly = [ {path: DIRS[0], stats: fs.lstatSync(DIRS[0])}, {path: DIRS[1], stats: fs.lstatSync(DIRS[1])}, {path: DIRS[2], stats: fs.lstatSync(DIRS[2])}, {path: DIRS[3], stats: fs.lstatSync(DIRS[3])} ] - var actualDirs = klawSync(FIXTURES_DIR, {nofile: true}) - assert.equal(actualDirs.length, expectedItems.length) - actualDirs.forEach(function (elem, i) { - assert.deepEqual(elem, expectedItems[i]) - assert.strictEqual(elem.path, expectedItems[i].path) - assert.deepEqual(elem.stats, expectedItems[i].stats) + const dirs = klawSync(TEST_DIR, {nofile: true}) + assert.equal(dirs.length, dirsOnly.length) + dirs.forEach((dir, i) => { + assert.deepEqual(dir, dirsOnly[i]) + assert.strictEqual(dir.path, dirsOnly[i].path) + assert.deepEqual(dir.stats, dirsOnly[i].stats) }) - done() }) - it('should ignore if opts.ignore is path name', function (done) { - var dirToIgnore = path.join(FIXTURES_DIR, 'node_modules') + it('should ignore if opts.ignore is path name', () => { + const dirToIgnore = path.join(TEST_DIR, 'node_modules') fs.ensureDirSync(dirToIgnore) - var expectedItems = [ - {path: DIRS[0], stats: fs.lstatSync(DIRS[0])}, - {path: FILES[0], stats: fs.lstatSync(FILES[0])}, - {path: DIRS[1], stats: fs.lstatSync(DIRS[1])}, - {path: DIRS[2], stats: fs.lstatSync(DIRS[2])}, - {path: DIRS[3], stats: fs.lstatSync(DIRS[3])}, - {path: FILES[1], stats: fs.lstatSync(FILES[1])}, - {path: FILES[2], stats: fs.lstatSync(FILES[2])} + const paths = [ + {path: DIRS[0], stats: fs.statSync(DIRS[0])}, + {path: FILES[0], stats: fs.statSync(FILES[0])}, + {path: DIRS[1], stats: fs.statSync(DIRS[1])}, + {path: DIRS[2], stats: fs.statSync(DIRS[2])}, + {path: DIRS[3], stats: fs.statSync(DIRS[3])}, + {path: FILES[1], stats: fs.statSync(FILES[1])}, + {path: FILES[2], stats: fs.statSync(FILES[2])} ] - var items = klawSync(FIXTURES_DIR, {ignore: 'node_modules'}) - assert.equal(items.length, expectedItems.length) - items.forEach(function (elem, i) { - assert.deepEqual(elem, expectedItems[i]) - assert.strictEqual(elem.path, expectedItems[i].path) - assert.deepEqual(elem.stats, expectedItems[i].stats) + const items = klawSync(TEST_DIR, {ignore: '**/node_modules'}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) }) - fs.removeSync(dirToIgnore) - done() }) - it('should ignore if opts.ignore is glob pattern', function (done) { - var dirToIgnore1 = path.join(FIXTURES_DIR, 'node_modules') - var dirToIgnore2 = path.join(FIXTURES_DIR, '.git') + it('should ignore if opts.ignore is glob pattern', () => { + const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') + const dirToIgnore2 = path.join(TEST_DIR, '.git') fs.ensureDirSync(dirToIgnore1) fs.ensureDirSync(dirToIgnore2) - var expectedItems = [ - {path: DIRS[0], stats: fs.lstatSync(DIRS[0])}, - {path: FILES[0], stats: fs.lstatSync(FILES[0])}, - {path: DIRS[1], stats: fs.lstatSync(DIRS[1])}, - {path: DIRS[2], stats: fs.lstatSync(DIRS[2])}, - {path: DIRS[3], stats: fs.lstatSync(DIRS[3])}, - {path: FILES[1], stats: fs.lstatSync(FILES[1])}, - {path: FILES[2], stats: fs.lstatSync(FILES[2])} + const paths = [ + {path: DIRS[0], stats: fs.statSync(DIRS[0])}, + {path: FILES[0], stats: fs.statSync(FILES[0])}, + {path: DIRS[1], stats: fs.statSync(DIRS[1])}, + {path: DIRS[2], stats: fs.statSync(DIRS[2])}, + {path: DIRS[3], stats: fs.statSync(DIRS[3])}, + {path: FILES[1], stats: fs.statSync(FILES[1])}, + {path: FILES[2], stats: fs.statSync(FILES[2])} ] - var items = klawSync(FIXTURES_DIR, {ignore: '{node_modules,.git}'}) - assert.equal(items.length, expectedItems.length) - items.forEach(function (elem, i) { - assert.deepEqual(elem, expectedItems[i]) - assert.strictEqual(elem.path, expectedItems[i].path) - assert.deepEqual(elem.stats, expectedItems[i].stats) + const items = klawSync(TEST_DIR, {ignore: '{**/node_modules,**/.git}'}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) }) - fs.removeSync(dirToIgnore1) - fs.removeSync(dirToIgnore2) - done() }) - it('should ignore if opts.ignore is array of items', function (done) { - var dirToIgnore1 = path.join(FIXTURES_DIR, 'node_modules') - var dirToIgnore2 = path.join(FIXTURES_DIR, '.git') - var fileToIgnore1 = path.join(FIXTURES_DIR, 'dir1', 'somefile.md') - var fileToIgnore2 = path.join(FIXTURES_DIR, 'dir2/dir2_1', 'someotherfile.md') + it('should ignore if opts.ignore is array of patterns', () => { + const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') + const dirToIgnore2 = path.join(TEST_DIR, '.git') + const fileToIgnore1 = path.join(TEST_DIR, 'dir1', 'somefile.md') + const fileToIgnore2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'someotherfile.md') fs.ensureDirSync(dirToIgnore1) fs.ensureDirSync(dirToIgnore2) fs.ensureFileSync(fileToIgnore1) fs.ensureFileSync(fileToIgnore2) - var expectedItems = [ - {path: DIRS[0], stats: fs.lstatSync(DIRS[0])}, - {path: FILES[0], stats: fs.lstatSync(FILES[0])}, - {path: DIRS[1], stats: fs.lstatSync(DIRS[1])}, - {path: DIRS[2], stats: fs.lstatSync(DIRS[2])}, - {path: DIRS[3], stats: fs.lstatSync(DIRS[3])}, - {path: FILES[1], stats: fs.lstatSync(FILES[1])}, - {path: FILES[2], stats: fs.lstatSync(FILES[2])} + const paths = [ + {path: DIRS[0], stats: fs.statSync(DIRS[0])}, + {path: FILES[0], stats: fs.statSync(FILES[0])}, + {path: DIRS[1], stats: fs.statSync(DIRS[1])}, + {path: DIRS[2], stats: fs.statSync(DIRS[2])}, + {path: DIRS[3], stats: fs.statSync(DIRS[3])}, + {path: FILES[1], stats: fs.statSync(FILES[1])}, + {path: FILES[2], stats: fs.statSync(FILES[2])} + ] + const items = klawSync(TEST_DIR, {ignore: ['**/node_modules', '**/.git', '**/*.md']}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) + }) + + it('should ignore if opts.ignore is glob pattern with negation', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.ensureFileSync(f1) + fs.ensureFileSync(f2) + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} + ] + const items = klawSync(TEST_DIR, {ignore: '!**/*.js'}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) + }) + + it('should filter if opts.filter is true based on path', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.ensureFileSync(f1) + fs.ensureFileSync(f2) + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} + ] + const filterFunc = (i) => path.extname(i.path) === '.js' + const items = klawSync(TEST_DIR, {filter: filterFunc}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) + }) + + it('should filter if opts.filter is true based on stats', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.outputFileSync(f1, 'test file 1 contents') + fs.outputFileSync(f2, 'test file 2 contents') + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} + ] + const filterFunc = (i) => i.stats.isFile() && i.stats.size > 0 + const items = klawSync(TEST_DIR, {filter: filterFunc}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) + }) + + it('should filter if opts.filter is true based on both path and stats', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.outputFileSync(f1, 'test file 1 contents') + fs.outputFileSync(f2, 'test file 2 contents') + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} ] - var items = klawSync(FIXTURES_DIR, {ignore: ['{node_modules,.git}', '*.md']}) - assert.equal(items.length, expectedItems.length) - items.forEach(function (elem, i) { - assert.deepEqual(elem, expectedItems[i]) - assert.strictEqual(elem.path, expectedItems[i].path) - assert.deepEqual(elem.stats, expectedItems[i].stats) + const filterFunc = (i) => path.extname(i.path) === '.js' && i.stats.size > 0 + const items = klawSync(TEST_DIR, {filter: filterFunc}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) }) - fs.removeSync(dirToIgnore1) - fs.removeSync(dirToIgnore2) - fs.removeSync(fileToIgnore1) - fs.removeSync(fileToIgnore2) - done() }) }) From 739a6e3c71edd6adb2c2d044534b2e68ccce7833 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sat, 22 Apr 2017 23:26:22 -0700 Subject: [PATCH 2/9] remove ignore option, remove micromatch, refactor filter option, update bm.js --- benchmark/bm.js | 60 +++++++++------------ klaw-sync.js | 19 +++---- package.json | 1 + test/test.js | 136 ++++++++++++++++++++++++++++-------------------- 4 files changed, 117 insertions(+), 99 deletions(-) diff --git a/benchmark/bm.js b/benchmark/bm.js index cc41a17..511b024 100644 --- a/benchmark/bm.js +++ b/benchmark/bm.js @@ -8,26 +8,24 @@ const klawSync = require('../klaw-sync.js') function help () { console.log(`Usage examples:\n`) - console.log(`npm run benchmark -- --dir= (basic usage without anything to ignore)`) - console.log(`npm run benchmark -- --dir= -i "{node_modules,.git}" (ignore node_modules and .git directories)`) - console.log(`npm run benchmark -- --dir= -i "node_modules" -i "*.js" (ignore node_modules and all js files)`) + console.log(`npm run benchmark -- --dir=`) + console.log(`npm run benchmark -- --dir= --nodir=true (ignore all directories)`) } -function perf (root, ign) { - var suite = Benchmark.Suite() - if (ign) { +function run (root, opts) { + if (!opts) { + const suite = Benchmark.Suite() suite.add('walk-sync', function () { - walkSync(root, {ignore: ign}) + walkSync(root) }).add('glob.sync', function () { globSync('**', { cwd: root, dot: true, mark: true, - strict: true, - ignore: ign + strict: true }) }).add('klaw-sync', function () { - klawSync(root, {ignore: ign}) + klawSync(root) }).on('error', function (er) { return er }).on('cycle', function (ev) { @@ -36,17 +34,19 @@ function perf (root, ign) { console.log('\nSummary: Fastest is ' + this.filter('fastest').map('name')) }).run({ 'async': false }) } else { + const suite = Benchmark.Suite() suite.add('walk-sync', function () { - walkSync(root) + walkSync(root, {directories: false}) }).add('glob.sync', function () { globSync('**', { cwd: root, dot: true, mark: true, - strict: true + strict: true, + nodir: true }) }).add('klaw-sync', function () { - klawSync(root) + klawSync(root, {nodir: true}) }).on('error', function (er) { return er }).on('cycle', function (ev) { @@ -57,28 +57,18 @@ function perf (root, ign) { } } -try { - if (!argv.dir) { - console.log('err: root dir must be specified.') - help() - process.exit(1) - } - var dir = path.resolve(argv.dir) - console.log('Running benchmark tests...\n') - console.log('root dir: ', argv.dir) - if (argv.i) { - process.stdout.write('ignore: ') - console.dir(argv.i) - console.log() - // convert ignore args to array - if (typeof argv.i === 'string') { - perf(dir, [argv.i]) - } else { - perf(dir, argv.i) - } +if (!argv.dir) { + console.log('err: root dir cannot be null.') + help() +} else { + const dir = path.resolve(argv.dir) + console.log('Running benchmark tests..') + if (argv.nodir) { + console.log(`root dir: ${dir}`) + console.log('option.nodir: true\n') + run(dir, {nodir: true}) } else { - perf(dir) + console.log(`root dir: ${dir}\n`) + run(dir) } -} catch (er) { - throw er } diff --git a/klaw-sync.js b/klaw-sync.js index 1bf5825..f7456da 100644 --- a/klaw-sync.js +++ b/klaw-sync.js @@ -1,6 +1,5 @@ 'use strict' const path = require('path') -const mm = require('micromatch') let fs try { fs = require('graceful-fs') @@ -14,20 +13,22 @@ function klawSync (dir, opts, ls) { const item = {path: pathItem, stats: stat} if (stat.isDirectory()) { if (!opts.nodir) { - if (opts.ignore) { - if (mm(pathItem, opts.ignore).length === 0) ls.push(item) - } else if (opts.filter) { - if (opts.filter(item)) ls.push(item) + if (opts.filter) { + if (opts.filter(item)) { + ls.push(item) + ls = klawSync(pathItem, opts, ls) + } + if (!opts.noRecursiveOnFilter) ls = klawSync(pathItem, opts, ls) } else { ls.push(item) + ls = klawSync(pathItem, opts, ls) } + } else { + ls = klawSync(pathItem, opts, ls) } - ls = klawSync(pathItem, opts, ls) } else { if (!opts.nofile) { - if (opts.ignore) { - if (mm(pathItem, opts.ignore).length === 0) ls.push(item) - } else if (opts.filter) { + if (opts.filter) { if (opts.filter(item)) ls.push(item) } else { ls.push(item) diff --git a/package.json b/package.json index d6af143..16ce01e 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ }, "homepage": "https://github.com/manidlou/node-klaw-sync#readme", "dependencies": { + "benchmark": "^2.1.4", "micromatch": "^2.3.11" }, "devDependencies": { diff --git a/test/test.js b/test/test.js index a23a912..1205d28 100644 --- a/test/test.js +++ b/test/test.js @@ -87,7 +87,7 @@ describe('klaw-sync', () => { assert.deepEqual(dir.stats, dirsOnly[i].stats) }) }) - + /* it('should ignore if opts.ignore is path name', () => { const dirToIgnore = path.join(TEST_DIR, 'node_modules') fs.ensureDirSync(dirToIgnore) @@ -100,7 +100,7 @@ describe('klaw-sync', () => { {path: FILES[1], stats: fs.statSync(FILES[1])}, {path: FILES[2], stats: fs.statSync(FILES[2])} ] - const items = klawSync(TEST_DIR, {ignore: '**/node_modules'}) + const items = klawSync(TEST_DIR, {ignore: 'node_modules'}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) @@ -123,7 +123,7 @@ describe('klaw-sync', () => { {path: FILES[1], stats: fs.statSync(FILES[1])}, {path: FILES[2], stats: fs.statSync(FILES[2])} ] - const items = klawSync(TEST_DIR, {ignore: '{**/node_modules,**/.git}'}) + const items = klawSync(TEST_DIR, {ignore: '{node_modules,.git}'}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) @@ -150,7 +150,7 @@ describe('klaw-sync', () => { {path: FILES[1], stats: fs.statSync(FILES[1])}, {path: FILES[2], stats: fs.statSync(FILES[2])} ] - const items = klawSync(TEST_DIR, {ignore: ['**/node_modules', '**/.git', '**/*.md']}) + const items = klawSync(TEST_DIR, {ignore: ['node_modules', '.git', '.md']}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) @@ -168,7 +168,7 @@ describe('klaw-sync', () => { {path: f1, stats: fs.statSync(f1)}, {path: f2, stats: fs.statSync(f2)} ] - const items = klawSync(TEST_DIR, {ignore: '!**/*.js'}) + const items = klawSync(TEST_DIR, {ignore: '.js'}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) @@ -176,61 +176,87 @@ describe('klaw-sync', () => { assert.deepEqual(p.stats, paths[i].stats) }) }) + */ - it('should filter if opts.filter is true based on path', () => { - const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') - const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') - fs.ensureFileSync(f1) - fs.ensureFileSync(f2) - const paths = [ - {path: f1, stats: fs.statSync(f1)}, - {path: f2, stats: fs.statSync(f2)} - ] - const filterFunc = (i) => path.extname(i.path) === '.js' - const items = klawSync(TEST_DIR, {filter: filterFunc}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) + describe('when opts.filter is true', () => { + it('should filter based on path', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.ensureFileSync(f1) + fs.ensureFileSync(f2) + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} + ] + const filterFunc = (i) => path.extname(i.path) === '.js' + const items = klawSync(TEST_DIR, {filter: filterFunc}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) }) - }) - it('should filter if opts.filter is true based on stats', () => { - const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') - const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') - fs.outputFileSync(f1, 'test file 1 contents') - fs.outputFileSync(f2, 'test file 2 contents') - const paths = [ - {path: f1, stats: fs.statSync(f1)}, - {path: f2, stats: fs.statSync(f2)} - ] - const filterFunc = (i) => i.stats.isFile() && i.stats.size > 0 - const items = klawSync(TEST_DIR, {filter: filterFunc}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) + it('should filter based on stats', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.outputFileSync(f1, 'test file 1 contents') + fs.outputFileSync(f2, 'test file 2 contents') + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} + ] + const filterFunc = (i) => i.stats.isFile() && i.stats.size > 0 + const items = klawSync(TEST_DIR, {filter: filterFunc}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) }) - }) - it('should filter if opts.filter is true based on both path and stats', () => { - const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') - const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') - fs.outputFileSync(f1, 'test file 1 contents') - fs.outputFileSync(f2, 'test file 2 contents') - const paths = [ - {path: f1, stats: fs.statSync(f1)}, - {path: f2, stats: fs.statSync(f2)} - ] - const filterFunc = (i) => path.extname(i.path) === '.js' && i.stats.size > 0 - const items = klawSync(TEST_DIR, {filter: filterFunc}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) + it('should filter based on both path and stats', () => { + const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') + const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') + fs.outputFileSync(f1, 'test file 1 contents') + fs.outputFileSync(f2, 'test file 2 contents') + const paths = [ + {path: f1, stats: fs.statSync(f1)}, + {path: f2, stats: fs.statSync(f2)} + ] + const filterFunc = (i) => path.extname(i.path) === '.js' && i.stats.size > 0 + const items = klawSync(TEST_DIR, {filter: filterFunc}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) + }) + + it('should filter but not recurse if noRecursiveOnFilter is true', () => { + const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') + const dirToIgnore2 = path.join(dirToIgnore1, 'somepkg') + fs.ensureDirSync(dirToIgnore2) + const paths = [ + {path: DIRS[0], stats: fs.statSync(DIRS[0])}, + {path: FILES[0], stats: fs.statSync(FILES[0])}, + {path: DIRS[1], stats: fs.statSync(DIRS[1])}, + {path: DIRS[2], stats: fs.statSync(DIRS[2])}, + {path: DIRS[3], stats: fs.statSync(DIRS[3])}, + {path: FILES[1], stats: fs.statSync(FILES[1])}, + {path: FILES[2], stats: fs.statSync(FILES[2])} + ] + const filterFunc = i => i.path.indexOf('node_modules') < 0 + const items = klawSync(TEST_DIR, {filter: filterFunc, noRecursiveOnFilter: true}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) }) }) }) From e56432d7a920c8516cdf875243ee0309fb64e75f Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 01:05:26 -0700 Subject: [PATCH 3/9] add more tests for filter, update readme --- README.md | 74 ++++++++++++++++++--------------- test/test.js | 114 +++++++++++---------------------------------------- 2 files changed, 66 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 3d4742a..7109b20 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -klaw-sync -========= +node-klaw-sync +============== [![npm Package](https://img.shields.io/npm/v/klaw-sync.svg?style=flat-square)](https://www.npmjs.com/package/klaw-sync) [![Build Status](https://travis-ci.org/manidlou/node-klaw-sync.svg?branch=master)](https://travis-ci.org/manidlou/node-klaw-sync) @@ -9,6 +9,10 @@ klaw-sync `klaw-sync` is a Node.js recursive file system walker, which is the synchronous counterpart of [klaw](https://github.com/jprichardson/node-klaw). It lists all files and directories inside a directory recursively and returns an array of objects that each object has two properties: `path` and `stats`. `path` is the full path of the file or directory and `stats` is an instance of [fs.Stats](https://nodejs.org/api/fs.html#fs_class_fs_stats). +Breaking Change +--------------- +`ignore` option is no longer supported (as of `v2.0.0`). Instead, you can use `filter` option to achieve the same functionality. Please see [examples](#examples). + Install ------- @@ -19,29 +23,30 @@ Usage ### klawSync(directory[, options]) -- `directory` `{String}` -- `options` `{Object}` *optional* (all options are `false` by default) - - `ignore` `{String | Array}` any paths or [micromatch](https://github.com/jonschlinkert/micromatch#features) patterns to ignore (can be string or an array of strings) - - `nodir` `{Boolean}` return only files (ignore directories) - - `nofile` `{Boolean}` return only directories (ignore files) +- `directory` `` +- `options` `` *optional* (all options are `false` by default) + - `nodir` `` return only files (ignore directories) + - `nofile` `` return only directories (ignore files) + - `noRecursiveOnFilter` `` if your `filter` function applies to **directories only**, set `true` to prevent unnecessary traversal of unwanted directories + - `filter` `` function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item -- return: `{Array}` `[{path: '', stats: {}}]` +- return: `>` `[{path: '', stats: {}}]` Examples -------- ```js -var klawSync = require('klaw-sync') -var paths = klawSync('/some/dir') +const klawSync = require('klaw-sync') +const paths = klawSync('/some/dir') // paths = [{path: '/some/dir/dir1', stats: {}}, {path: '/some/dir/file1', stats: {}}] ``` _**catch error**_ ```js -var klawSync = require('klaw-sync') +const klawSync = require('klaw-sync') -var paths +let paths try { paths = klawSync('/some/dir') } catch (er) { @@ -53,38 +58,48 @@ console.dir(paths) _**files only**_ ```js -var klawSync = require('klaw-sync') -var files = klawSync('/some/dir', {nodir: true}) +const klawSync = require('klaw-sync') +const files = klawSync('/some/dir', {nodir: true}) // files = [{path: '/some/dir/file1', stats: {}}, {path: '/some/dir/file2', stats: {}}] ``` _**directories only**_ ```js -var klawSync = require('klaw-sync') -var dirs = klawSync('/some/dir', {nofile: true}) +const klawSync = require('klaw-sync') +const dirs = klawSync('/some/dir', {nofile: true}) // dirs = [{path: '/some/dir/dir1', stats: {}}, {path: '/some/dir/dir2', stats: {}}] ``` _**ignore `node_modules`**_ +Notice here because the `filter` function is applied to directories only, `noRecursiveOnFilter: true` is used since we don't want anything from `node_modules` in this case. + ```js -var klawSync = require('klaw-sync') -var paths = klawSync('/some/dir', {ignore: 'node_modules'}) +const klawSync = require('klaw-sync') + +const filterFn = item => item.path.indexOf('node_modules') < 0 +const paths = klawSync('/some/dir', { filter: filterFn, noRecursiveOnFilter: true }) ``` -_**ignore `node_modules` and `.git` using [micromatch](https://github.com/jonschlinkert/micromatch#features) patterns**_ +_**ignore `node_modules` and `.git`**_ ```js -var klawSync = require('klaw-sync') -var paths = klawSync('/some/dir', {ignore: '{node_modules,.git}'}) +const klawSync = require('klaw-sync') + +const filterFn = item => item.path.indexOf('node_modules') < 0 && item.path.indexOf('.git') < 0 +const paths = klawSync('/some/dir', { filter: filterFn, noRecursiveOnFilter: true }) ``` -_**ignore `node_modules`, `.git` and all `*.js` files using [micromatch](https://github.com/jonschlinkert/micromatch#features) patterns**_ +_**get all `js` files**_ + +Notice here `noRecursiveOnFilter` is not required since we are interested in all `js` files. In other words, although the `filter` function doesn't pass for directories, we still want to read them and see if they have any `js` files. ```js -var klawSync = require('klaw-sync') -var paths = klawSync('/some/dir', {ignore: ['{node_modules,.git}', '*.js']}) +const klawSync = require('klaw-sync') + +const filterFn = item => path.extname(item.path) === '.js' +const paths = klawSync('/some/dir', { filter: filterFn }) ``` Run tests @@ -100,7 +115,7 @@ lint & unit: `npm test` Performance compare to other similar modules ----------------------------------------------- -The `bm.js` runs some basic [benchmark](https://github.com/bestiejs/benchmark.js) tests for two cases, `without --ignore` (basic usage) and `with --ignore`, on these modules: +The `bm.js` runs some basic [benchmark](https://github.com/bestiejs/benchmark.js) tests for two cases: basic usage and with `--nodir=true` (get files only), on these modules: - `klaw-sync` - [walk-sync](https://github.com/joliss/node-walk-sync) @@ -108,16 +123,11 @@ The `bm.js` runs some basic [benchmark](https://github.com/bestiejs/benchmark.js Just for fun, it turned out (as of January 25, 2017) for the most cases `klaw-sync` is faster than other modules! -#####run benchmark - -To run benchmark, just specify the root `--dir=`. To ignore paths or patterns, use `-i` flag. +##### run benchmark `npm run benchmark -- --dir=/some/dir` -`npm run benchmark -- --dir=/some/dir -i "node_modules"` - -`npm run benchmark -- --dir=/some/dir -i "node_modules" -i "*.js"` - +`npm run benchmark -- --dir=/some/dir --nodir=true` Credit ------ diff --git a/test/test.js b/test/test.js index 1205d28..be24c67 100644 --- a/test/test.js +++ b/test/test.js @@ -87,96 +87,6 @@ describe('klaw-sync', () => { assert.deepEqual(dir.stats, dirsOnly[i].stats) }) }) - /* - it('should ignore if opts.ignore is path name', () => { - const dirToIgnore = path.join(TEST_DIR, 'node_modules') - fs.ensureDirSync(dirToIgnore) - const paths = [ - {path: DIRS[0], stats: fs.statSync(DIRS[0])}, - {path: FILES[0], stats: fs.statSync(FILES[0])}, - {path: DIRS[1], stats: fs.statSync(DIRS[1])}, - {path: DIRS[2], stats: fs.statSync(DIRS[2])}, - {path: DIRS[3], stats: fs.statSync(DIRS[3])}, - {path: FILES[1], stats: fs.statSync(FILES[1])}, - {path: FILES[2], stats: fs.statSync(FILES[2])} - ] - const items = klawSync(TEST_DIR, {ignore: 'node_modules'}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) - }) - }) - - it('should ignore if opts.ignore is glob pattern', () => { - const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') - const dirToIgnore2 = path.join(TEST_DIR, '.git') - fs.ensureDirSync(dirToIgnore1) - fs.ensureDirSync(dirToIgnore2) - const paths = [ - {path: DIRS[0], stats: fs.statSync(DIRS[0])}, - {path: FILES[0], stats: fs.statSync(FILES[0])}, - {path: DIRS[1], stats: fs.statSync(DIRS[1])}, - {path: DIRS[2], stats: fs.statSync(DIRS[2])}, - {path: DIRS[3], stats: fs.statSync(DIRS[3])}, - {path: FILES[1], stats: fs.statSync(FILES[1])}, - {path: FILES[2], stats: fs.statSync(FILES[2])} - ] - const items = klawSync(TEST_DIR, {ignore: '{node_modules,.git}'}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) - }) - }) - - it('should ignore if opts.ignore is array of patterns', () => { - const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') - const dirToIgnore2 = path.join(TEST_DIR, '.git') - const fileToIgnore1 = path.join(TEST_DIR, 'dir1', 'somefile.md') - const fileToIgnore2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'someotherfile.md') - fs.ensureDirSync(dirToIgnore1) - fs.ensureDirSync(dirToIgnore2) - fs.ensureFileSync(fileToIgnore1) - fs.ensureFileSync(fileToIgnore2) - const paths = [ - {path: DIRS[0], stats: fs.statSync(DIRS[0])}, - {path: FILES[0], stats: fs.statSync(FILES[0])}, - {path: DIRS[1], stats: fs.statSync(DIRS[1])}, - {path: DIRS[2], stats: fs.statSync(DIRS[2])}, - {path: DIRS[3], stats: fs.statSync(DIRS[3])}, - {path: FILES[1], stats: fs.statSync(FILES[1])}, - {path: FILES[2], stats: fs.statSync(FILES[2])} - ] - const items = klawSync(TEST_DIR, {ignore: ['node_modules', '.git', '.md']}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) - }) - }) - - it('should ignore if opts.ignore is glob pattern with negation', () => { - const f1 = path.join(TEST_DIR, 'dir1', 'foo.js') - const f2 = path.join(TEST_DIR, 'dir2', 'dir2_1', 'bar.js') - fs.ensureFileSync(f1) - fs.ensureFileSync(f2) - const paths = [ - {path: f1, stats: fs.statSync(f1)}, - {path: f2, stats: fs.statSync(f2)} - ] - const items = klawSync(TEST_DIR, {ignore: '.js'}) - assert.equal(items.length, paths.length) - items.forEach((p, i) => { - assert.deepEqual(p, paths[i]) - assert.strictEqual(p.path, paths[i].path) - assert.deepEqual(p.stats, paths[i].stats) - }) - }) - */ describe('when opts.filter is true', () => { it('should filter based on path', () => { @@ -258,5 +168,29 @@ describe('klaw-sync', () => { assert.deepEqual(p.stats, paths[i].stats) }) }) + + it('should filter when it is used to ignore items', () => { + const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') + const dirToIgnore2 = path.join(TEST_DIR, '.git') + fs.ensureDirSync(dirToIgnore1) + fs.ensureDirSync(dirToIgnore2) + const paths = [ + {path: DIRS[0], stats: fs.statSync(DIRS[0])}, + {path: FILES[0], stats: fs.statSync(FILES[0])}, + {path: DIRS[1], stats: fs.statSync(DIRS[1])}, + {path: DIRS[2], stats: fs.statSync(DIRS[2])}, + {path: DIRS[3], stats: fs.statSync(DIRS[3])}, + {path: FILES[1], stats: fs.statSync(FILES[1])}, + {path: FILES[2], stats: fs.statSync(FILES[2])} + ] + const filterFunc = i => i.path.indexOf('node_modules') < 0 && i.path.indexOf('.git') < 0 + const items = klawSync(TEST_DIR, {filter: filterFunc, noRecursiveOnFilter: true}) + assert.equal(items.length, paths.length) + items.forEach((p, i) => { + assert.deepEqual(p, paths[i]) + assert.strictEqual(p.path, paths[i].path) + assert.deepEqual(p.stats, paths[i].stats) + }) + }) }) }) From dd27335b8970cd5e8d93389830e51be046ebfba1 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 12:04:00 -0700 Subject: [PATCH 4/9] refactor filter additional option, update readme --- README.md | 28 +++++++++++++++++++--------- klaw-sync.js | 2 +- test/test.js | 6 +++--- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7109b20..e9cac39 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,6 @@ node-klaw-sync `klaw-sync` is a Node.js recursive file system walker, which is the synchronous counterpart of [klaw](https://github.com/jprichardson/node-klaw). It lists all files and directories inside a directory recursively and returns an array of objects that each object has two properties: `path` and `stats`. `path` is the full path of the file or directory and `stats` is an instance of [fs.Stats](https://nodejs.org/api/fs.html#fs_class_fs_stats). -Breaking Change ---------------- -`ignore` option is no longer supported (as of `v2.0.0`). Instead, you can use `filter` option to achieve the same functionality. Please see [examples](#examples). - Install ------- @@ -27,7 +23,7 @@ Usage - `options` `` *optional* (all options are `false` by default) - `nodir` `` return only files (ignore directories) - `nofile` `` return only directories (ignore files) - - `noRecursiveOnFilter` `` if your `filter` function applies to **directories only**, set `true` to prevent unnecessary traversal of unwanted directories + - `noRecurseOnFilter` `` the default behavior is to read all directories even though they don't pass the `filter` function (won't be included but still will be traversed). Set `true` to prevent unnecessary traversal of unwanted directories - `filter` `` function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item - return: `>` `[{path: '', stats: {}}]` @@ -73,13 +69,13 @@ const dirs = klawSync('/some/dir', {nofile: true}) _**ignore `node_modules`**_ -Notice here because the `filter` function is applied to directories only, `noRecursiveOnFilter: true` is used since we don't want anything from `node_modules` in this case. +Notice here `noRecurseOnFilter: true` is used since we don't want anything from `node_modules` in this case (no inclusion and no traversal). ```js const klawSync = require('klaw-sync') const filterFn = item => item.path.indexOf('node_modules') < 0 -const paths = klawSync('/some/dir', { filter: filterFn, noRecursiveOnFilter: true }) +const paths = klawSync('/some/dir', { filter: filterFn, noRecurseOnFilter: true }) ``` _**ignore `node_modules` and `.git`**_ @@ -88,20 +84,34 @@ _**ignore `node_modules` and `.git`**_ const klawSync = require('klaw-sync') const filterFn = item => item.path.indexOf('node_modules') < 0 && item.path.indexOf('.git') < 0 -const paths = klawSync('/some/dir', { filter: filterFn, noRecursiveOnFilter: true }) +const paths = klawSync('/some/dir', { filter: filterFn, noRecurseOnFilter: true }) ``` _**get all `js` files**_ -Notice here `noRecursiveOnFilter` is not required since we are interested in all `js` files. In other words, although the `filter` function doesn't pass for directories, we still want to read them and see if they have any `js` files. +Here `noRecurseOnFilter` is not required since we are interested in all `js` files. In other words, although no directories pass the `filter` function, we still want to read them and see if they have any `js` files. ```js +const path = require('path') const klawSync = require('klaw-sync') const filterFn = item => path.extname(item.path) === '.js' const paths = klawSync('/some/dir', { filter: filterFn }) ``` +_**filter based on stats**_ + +Again here `noRecurseOnFilter` is not required since we still want to read all directories even though they don't pass the `filter` function, to see if their contents pass the `filter` function. + +```js +const klawSync = require('klaw-sync') + +const refTime = new Date(2017, 3, 24).getTime() + +const filterFn = item => item.stats.mtime > refTime +const paths = klawSync('/some/dir', { filter: filterFn }) +``` + Run tests --------- diff --git a/klaw-sync.js b/klaw-sync.js index f7456da..8c5737b 100644 --- a/klaw-sync.js +++ b/klaw-sync.js @@ -18,7 +18,7 @@ function klawSync (dir, opts, ls) { ls.push(item) ls = klawSync(pathItem, opts, ls) } - if (!opts.noRecursiveOnFilter) ls = klawSync(pathItem, opts, ls) + if (!opts.noRecurseOnFilter) ls = klawSync(pathItem, opts, ls) } else { ls.push(item) ls = klawSync(pathItem, opts, ls) diff --git a/test/test.js b/test/test.js index be24c67..1296746 100644 --- a/test/test.js +++ b/test/test.js @@ -146,7 +146,7 @@ describe('klaw-sync', () => { }) }) - it('should filter but not recurse if noRecursiveOnFilter is true', () => { + it('should filter but not recurse if noRecurseOnFilter is true', () => { const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') const dirToIgnore2 = path.join(dirToIgnore1, 'somepkg') fs.ensureDirSync(dirToIgnore2) @@ -160,7 +160,7 @@ describe('klaw-sync', () => { {path: FILES[2], stats: fs.statSync(FILES[2])} ] const filterFunc = i => i.path.indexOf('node_modules') < 0 - const items = klawSync(TEST_DIR, {filter: filterFunc, noRecursiveOnFilter: true}) + const items = klawSync(TEST_DIR, {filter: filterFunc, noRecurseOnFilter: true}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) @@ -184,7 +184,7 @@ describe('klaw-sync', () => { {path: FILES[2], stats: fs.statSync(FILES[2])} ] const filterFunc = i => i.path.indexOf('node_modules') < 0 && i.path.indexOf('.git') < 0 - const items = klawSync(TEST_DIR, {filter: filterFunc, noRecursiveOnFilter: true}) + const items = klawSync(TEST_DIR, {filter: filterFunc, noRecurseOnFilter: true}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) From 56d1828f99cae41189ae329d17fc12c362bd0291 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 12:21:58 -0700 Subject: [PATCH 5/9] update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1604abd..b102000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +2.0.0 / 2017-04-23 +------------------ + +### Removed +- **BREAKING:** Removed support for `ignore` option due to inconsistency in glob pattern usages and relatively poor performance. See: [#1] + +### Added +- `filter` option. A function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item. +- `noRecurseOnFilter` option to prevent unnecessary traversal of unwanted directories when `filter` function is used. + 1.1.2 / 2017-02-17 ------------------ From 2389ab011ef74f9e242b31a14a099bc556df7f00 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 12:37:31 -0700 Subject: [PATCH 6/9] update readme and changelog --- CHANGELOG.md | 2 ++ README.md | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b102000..3dcd46a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,3 +29,5 @@ - initial release +[#1]: https://github.com/manidlou/node-klaw-sync/issues/1 "loading all files with certain name" + diff --git a/README.md b/README.md index e9cac39..e5b5f30 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Usage - `options` `` *optional* (all options are `false` by default) - `nodir` `` return only files (ignore directories) - `nofile` `` return only directories (ignore files) - - `noRecurseOnFilter` `` the default behavior is to read all directories even though they don't pass the `filter` function (won't be included but still will be traversed). Set `true` to prevent unnecessary traversal of unwanted directories + - `noRecurseOnFilter` `` when `filter` function is used, the default behavior is to read all directories even though they don't pass the `filter` function (won't be included but still will be traversed). Set `true` to prevent unnecessary traversal of unwanted directories - `filter` `` function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item - return: `>` `[{path: '', stats: {}}]` @@ -33,6 +33,7 @@ Examples ```js const klawSync = require('klaw-sync') + const paths = klawSync('/some/dir') // paths = [{path: '/some/dir/dir1', stats: {}}, {path: '/some/dir/file1', stats: {}}] ``` @@ -55,6 +56,7 @@ _**files only**_ ```js const klawSync = require('klaw-sync') + const files = klawSync('/some/dir', {nodir: true}) // files = [{path: '/some/dir/file1', stats: {}}, {path: '/some/dir/file2', stats: {}}] ``` @@ -63,6 +65,7 @@ _**directories only**_ ```js const klawSync = require('klaw-sync') + const dirs = klawSync('/some/dir', {nofile: true}) // dirs = [{path: '/some/dir/dir1', stats: {}}, {path: '/some/dir/dir2', stats: {}}] ``` @@ -107,8 +110,8 @@ Again here `noRecurseOnFilter` is not required since we still want to read all d const klawSync = require('klaw-sync') const refTime = new Date(2017, 3, 24).getTime() +const filterFn = item => item.stats.mtime.getTime() > refTime -const filterFn = item => item.stats.mtime > refTime const paths = klawSync('/some/dir', { filter: filterFn }) ``` @@ -131,7 +134,7 @@ The `bm.js` runs some basic [benchmark](https://github.com/bestiejs/benchmark.js - [walk-sync](https://github.com/joliss/node-walk-sync) - [glob.sync](https://github.com/isaacs/node-glob#globsyncpattern-options) -Just for fun, it turned out (as of January 25, 2017) for the most cases `klaw-sync` is faster than other modules! +It turned out (as of January 25, 2017) for the most cases `klaw-sync` is faster than other modules! ##### run benchmark From c4c78c5b35f9cb00661cd2e65f5bfc723ef1dde5 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 12:46:41 -0700 Subject: [PATCH 7/9] remove unnecessary deps --- package.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package.json b/package.json index 16ce01e..cc41b07 100644 --- a/package.json +++ b/package.json @@ -21,10 +21,6 @@ "url": "https://github.com/manidlou/node-klaw-sync/issues" }, "homepage": "https://github.com/manidlou/node-klaw-sync#readme", - "dependencies": { - "benchmark": "^2.1.4", - "micromatch": "^2.3.11" - }, "devDependencies": { "benchmark": "^2.1.3", "fs-extra": "^1.0.0", From e76afdf7fd9053e67d3da90ec59d2fcffa8a7537 Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 16:25:50 -0700 Subject: [PATCH 8/9] change option name, update readme and changelog --- CHANGELOG.md | 2 +- README.md | 27 +++++++++++++++++---------- benchmark/bm.js | 14 +++++++------- klaw-sync.js | 23 +++++++++-------------- package.json | 2 +- test/test.js | 6 +++--- 6 files changed, 38 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dcd46a..9b46f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ### Added - `filter` option. A function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item. -- `noRecurseOnFilter` option to prevent unnecessary traversal of unwanted directories when `filter` function is used. +- `noRecurseOnFailedFilter` option to prevent unnecessary traversal of unwanted directories when `filter` function is used. 1.1.2 / 2017-02-17 ------------------ diff --git a/README.md b/README.md index e5b5f30..3776183 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,15 @@ Usage ### klawSync(directory[, options]) - `directory` `` -- `options` `` *optional* (all options are `false` by default) - - `nodir` `` return only files (ignore directories) - - `nofile` `` return only directories (ignore files) - - `noRecurseOnFilter` `` when `filter` function is used, the default behavior is to read all directories even though they don't pass the `filter` function (won't be included but still will be traversed). Set `true` to prevent unnecessary traversal of unwanted directories - - `filter` `` function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item +- `options` `` (optional) _all options are `false` by default_ + - `nodir` `` + - return only files (ignore directories) + - `nofile` `` + - return only directories (ignore files) + - `noRecurseOnFailedFilter` `` + - when `filter` function is used, the default behavior is to read all directories even if they don't pass the `filter` function (won't be included but still will be traversed). If you set `true`, there will be neither inclusion nor traversal for directories that don't pass the `filter` function + - `filter` `` + - function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item - return: `>` `[{path: '', stats: {}}]` @@ -72,13 +76,14 @@ const dirs = klawSync('/some/dir', {nofile: true}) _**ignore `node_modules`**_ -Notice here `noRecurseOnFilter: true` is used since we don't want anything from `node_modules` in this case (no inclusion and no traversal). +Notice here `noRecurseOnFailedFilter: true` option is used since we don't want anything from `node_modules` (no inclusion and no traversal). ```js const klawSync = require('klaw-sync') const filterFn = item => item.path.indexOf('node_modules') < 0 -const paths = klawSync('/some/dir', { filter: filterFn, noRecurseOnFilter: true }) + +const paths = klawSync('/some/dir', { filter: filterFn, noRecurseOnFailedFilter: true }) ``` _**ignore `node_modules` and `.git`**_ @@ -87,24 +92,26 @@ _**ignore `node_modules` and `.git`**_ const klawSync = require('klaw-sync') const filterFn = item => item.path.indexOf('node_modules') < 0 && item.path.indexOf('.git') < 0 -const paths = klawSync('/some/dir', { filter: filterFn, noRecurseOnFilter: true }) + +const paths = klawSync('/some/dir', { filter: filterFn, noRecurseOnFailedFilter: true }) ``` _**get all `js` files**_ -Here `noRecurseOnFilter` is not required since we are interested in all `js` files. In other words, although no directories pass the `filter` function, we still want to read them and see if they have any `js` files. +Here `noRecurseOnFailedFilter` option is not required since we are interested in all `js` files. In other words, although no directories pass the `filter` function, we still want to read them and see if they have any `js` files. ```js const path = require('path') const klawSync = require('klaw-sync') const filterFn = item => path.extname(item.path) === '.js' + const paths = klawSync('/some/dir', { filter: filterFn }) ``` _**filter based on stats**_ -Again here `noRecurseOnFilter` is not required since we still want to read all directories even though they don't pass the `filter` function, to see if their contents pass the `filter` function. +Again here `noRecurseOnFailedFilter` option is not required since we still want to read all directories even though they don't pass the `filter` function, to see if their contents pass the `filter` function. ```js const klawSync = require('klaw-sync') diff --git a/benchmark/bm.js b/benchmark/bm.js index 511b024..8f7f4f7 100644 --- a/benchmark/bm.js +++ b/benchmark/bm.js @@ -12,7 +12,7 @@ function help () { console.log(`npm run benchmark -- --dir= --nodir=true (ignore all directories)`) } -function run (root, opts) { +function runBm (root, opts) { if (!opts) { const suite = Benchmark.Suite() suite.add('walk-sync', function () { @@ -31,8 +31,8 @@ function run (root, opts) { }).on('cycle', function (ev) { console.log(String(ev.target)) }).on('complete', function () { - console.log('\nSummary: Fastest is ' + this.filter('fastest').map('name')) - }).run({ 'async': false }) + console.log('Fastest is ' + this.filter('fastest').map('name')) + }).run() } else { const suite = Benchmark.Suite() suite.add('walk-sync', function () { @@ -52,8 +52,8 @@ function run (root, opts) { }).on('cycle', function (ev) { console.log(String(ev.target)) }).on('complete', function () { - console.log('\nSummary: Fastest is ' + this.filter('fastest').map('name')) - }).run({ 'async': false }) + console.log('Fastest is ' + this.filter('fastest').map('name')) + }).run() } } @@ -66,9 +66,9 @@ if (!argv.dir) { if (argv.nodir) { console.log(`root dir: ${dir}`) console.log('option.nodir: true\n') - run(dir, {nodir: true}) + runBm(dir, {nodir: true}) } else { console.log(`root dir: ${dir}\n`) - run(dir) + runBm(dir) } } diff --git a/klaw-sync.js b/klaw-sync.js index 8c5737b..36eaa03 100644 --- a/klaw-sync.js +++ b/klaw-sync.js @@ -12,27 +12,22 @@ function klawSync (dir, opts, ls) { const stat = fs.lstatSync(pathItem) const item = {path: pathItem, stats: stat} if (stat.isDirectory()) { - if (!opts.nodir) { - if (opts.filter) { - if (opts.filter(item)) { - ls.push(item) - ls = klawSync(pathItem, opts, ls) - } - if (!opts.noRecurseOnFilter) ls = klawSync(pathItem, opts, ls) - } else { + if (opts.filter) { + if (opts.filter(item)) { ls.push(item) ls = klawSync(pathItem, opts, ls) + } else { + if (!opts.noRecurseOnFailedFilter) ls = klawSync(pathItem, opts, ls) } } else { + if (!opts.nodir) ls.push(item) ls = klawSync(pathItem, opts, ls) } } else { - if (!opts.nofile) { - if (opts.filter) { - if (opts.filter(item)) ls.push(item) - } else { - ls.push(item) - } + if (opts.filter) { + if (opts.filter(item)) ls.push(item) + } else { + if (!opts.nofile) ls.push(item) } } } diff --git a/package.json b/package.json index cc41b07..827cb2a 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "homepage": "https://github.com/manidlou/node-klaw-sync#readme", "devDependencies": { - "benchmark": "^2.1.3", + "benchmark": "^2.1.4", "fs-extra": "^1.0.0", "glob": "^7.1.1", "minimist": "^1.2.0", diff --git a/test/test.js b/test/test.js index 1296746..f117e5c 100644 --- a/test/test.js +++ b/test/test.js @@ -146,7 +146,7 @@ describe('klaw-sync', () => { }) }) - it('should filter but not recurse if noRecurseOnFilter is true', () => { + it('should filter but not recurse if noRecurseOnFailedFilter is true', () => { const dirToIgnore1 = path.join(TEST_DIR, 'node_modules') const dirToIgnore2 = path.join(dirToIgnore1, 'somepkg') fs.ensureDirSync(dirToIgnore2) @@ -160,7 +160,7 @@ describe('klaw-sync', () => { {path: FILES[2], stats: fs.statSync(FILES[2])} ] const filterFunc = i => i.path.indexOf('node_modules') < 0 - const items = klawSync(TEST_DIR, {filter: filterFunc, noRecurseOnFilter: true}) + const items = klawSync(TEST_DIR, {filter: filterFunc, noRecurseOnFailedFilter: true}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) @@ -184,7 +184,7 @@ describe('klaw-sync', () => { {path: FILES[2], stats: fs.statSync(FILES[2])} ] const filterFunc = i => i.path.indexOf('node_modules') < 0 && i.path.indexOf('.git') < 0 - const items = klawSync(TEST_DIR, {filter: filterFunc, noRecurseOnFilter: true}) + const items = klawSync(TEST_DIR, {filter: filterFunc, noRecurseOnFailedFilter: true}) assert.equal(items.length, paths.length) items.forEach((p, i) => { assert.deepEqual(p, paths[i]) From 432367706a3b846b9992f075b1475fb925aa5e8c Mon Sep 17 00:00:00 2001 From: Mani Maghsoudlou Date: Sun, 23 Apr 2017 16:39:27 -0700 Subject: [PATCH 9/9] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3776183..0c01a57 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Usage - `filter` `` - function that gets one argument `fn({path: '', stats: {}})` and returns true to include or false to exclude the item -- return: `>` `[{path: '', stats: {}}]` +- **Return:** `>` `[{path: '', stats: {}}]` Examples --------