-
-
Notifications
You must be signed in to change notification settings - Fork 359
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Caching #101
Caching #101
Changes from 3 commits
f4be86f
7eec3e0
ed1a9ee
84fce89
3f9952c
1d625e8
9b014bb
1343d4a
ca01f07
f7694b4
f4b9f24
dcef2dc
03de5e7
1c08d5b
0f73158
afedbfb
0319449
b71f8c9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,5 @@ | ||
.nyc_output | ||
.nyc_cache | ||
coverage | ||
node_modules | ||
!node_modules/spawn-wrap | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,22 +11,34 @@ var onExit = require('signal-exit') | |
var stripBom = require('strip-bom') | ||
var SourceMapCache = require('./lib/source-map-cache') | ||
var resolveFrom = require('resolve-from') | ||
var crypto = require('crypto') | ||
|
||
function md5 (str) { | ||
return crypto | ||
.createHash('md5') | ||
.update(str, 'utf8') | ||
.digest('hex') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Personally I'd be happy with this on a single line. Is md5 still acceptable for these kind of hashes? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. https://blog.risingstack.com/automatic-cache-busting-for-your-css/
via There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. MD5 is totally fine for non-cryptographically stuff. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I couldn't find There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nice write-up, thanks. |
||
} | ||
|
||
/* istanbul ignore next */ | ||
if (/index\.covered\.js$/.test(__filename)) { | ||
require('./lib/self-coverage-helper') | ||
} | ||
|
||
function NYC (opts) { | ||
if (opts && opts.istanbul) { | ||
opts._istanbul = opts.istanbul | ||
delete opts.istanbul | ||
} | ||
_.extend(this, { | ||
subprocessBin: path.resolve( | ||
__dirname, | ||
'./bin/nyc.js' | ||
), | ||
tempDirectory: './.nyc_output', | ||
cacheDirectory: './.nyc_cache', | ||
cwd: process.env.NYC_CWD || process.cwd(), | ||
reporter: 'text', | ||
istanbul: require('istanbul'), | ||
sourceMapCache: new SourceMapCache(), | ||
require: [] | ||
}, opts) | ||
|
@@ -48,7 +60,6 @@ function NYC (opts) { | |
// require extensions can be provided as config in package.json. | ||
this.require = config.require ? config.require : this.require | ||
|
||
this.instrumenter = this._createInstrumenter() | ||
this._createOutputDirectory() | ||
} | ||
|
||
|
@@ -67,14 +78,18 @@ NYC.prototype._loadAdditionalModules = function () { | |
}) | ||
} | ||
|
||
NYC.prototype.instrumenter = function () { | ||
return this._instrumenter || (this._instrumenter = this._createInstrumenter()) | ||
} | ||
|
||
NYC.prototype._createInstrumenter = function () { | ||
var configFile = path.resolve(this.cwd, './.istanbul.yml') | ||
|
||
if (!fs.existsSync(configFile)) configFile = undefined | ||
|
||
var instrumenterConfig = this.istanbul.config.loadFile(configFile).instrumentation.config | ||
var instrumenterConfig = this.istanbul().config.loadFile(configFile).instrumentation.config | ||
|
||
return new this.istanbul.Instrumenter({ | ||
return new (this.istanbul()).Instrumenter({ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Assign |
||
coverageVariable: '__coverage__', | ||
embedSource: instrumenterConfig['embed-source'], | ||
noCompact: !instrumenterConfig.compact, | ||
|
@@ -104,7 +119,7 @@ NYC.prototype.addFile = function (filename, returnImmediately) { | |
|
||
if (instrument) { | ||
this.sourceMapCache.add(filename, content) | ||
content = this.instrumenter.instrumentSync(content, './' + relFile) | ||
content = this.instrumenter().instrumentSync(content, './' + relFile) | ||
} else if (returnImmediately) { | ||
return {} | ||
} | ||
|
@@ -132,7 +147,7 @@ NYC.prototype.addAllFiles = function () { | |
var obj = _this.addFile(filename, true) | ||
if (obj.instrument) { | ||
module._compile( | ||
_this.instrumenter.getPreamble(obj.content, obj.relFile), | ||
_this.instrumenter().getPreamble(obj.content, obj.relFile), | ||
filename | ||
) | ||
} | ||
|
@@ -153,8 +168,16 @@ NYC.prototype._wrapRequire = function () { | |
|
||
_this.sourceMapCache.add(filename, code) | ||
|
||
// now instrument the compiled code. | ||
return _this.instrumenter.instrumentSync(code, './' + relFile) | ||
var hash = md5(code) | ||
var cacheFilePath = path.join(_this._cacheDirectory(), hash + '.js') | ||
|
||
try { | ||
return fs.readFileSync(cacheFilePath, 'utf8') | ||
} catch (e) { | ||
var instrumented = _this.instrumenter().instrumentSync(code, './' + relFile) | ||
fs.writeFile(cacheFilePath, instrumented) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suppose it's a cache so you don't need to block the process nor do you care if it fails… maybe add a comment to that effect? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
That was my thought, but honestly I wasn't sure.
Sounds like a plan. Is there any danger to writing async though? Should I change it back? I do not know enough about how file system failures propogate. Also, as stated here, it doesn't seem cache collisions are happening at all in the tests I've run, so I'm not even sure what behavior it will have. Definitely needs some more testing. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd assume the writes are consecutive, not parallel, so even if there is a collision it just means the cache file is written a couple times. It'll still be a valid cache file in the end. Errors won't be reported though, e.g. if we somehow screw up writing the cache dir we'd never notice. OTOH if we fail to write the cache file once in a while that's not the end of the world (and a test run shouldn't fail because of it). I suppose this could use async with a `console.warn() if there are errors? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
I think we are going to run into a lot more write collisions when they start using Perhaps we could log There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Turns out |
||
return instrumented | ||
} | ||
}) | ||
} | ||
|
||
|
@@ -164,6 +187,7 @@ NYC.prototype.cleanup = function () { | |
|
||
NYC.prototype._createOutputDirectory = function () { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a bit of a misnomer now. |
||
mkdirp.sync(this.tmpDirectory()) | ||
mkdirp.sync(this._cacheDirectory()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, but we pass allow There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yea, fair enough. |
||
} | ||
|
||
NYC.prototype._wrapExit = function () { | ||
|
@@ -195,11 +219,15 @@ NYC.prototype.writeCoverageFile = function () { | |
) | ||
} | ||
|
||
NYC.prototype.istanbul = function () { | ||
return this._istanbul || (this._istanbul = require('istanbul')) | ||
} | ||
|
||
NYC.prototype.report = function (cb, _collector, _reporter) { | ||
cb = cb || function () {} | ||
|
||
var collector = _collector || new this.istanbul.Collector() | ||
var reporter = _reporter || new this.istanbul.Reporter() | ||
var collector = _collector || new (this.istanbul()).Collector() | ||
var reporter = _reporter || new (this.istanbul()).Reporter() | ||
|
||
this._loadReports().forEach(function (report) { | ||
collector.add(report) | ||
|
@@ -232,6 +260,10 @@ NYC.prototype.tmpDirectory = function () { | |
return path.resolve(this.cwd, './', this.tempDirectory) | ||
} | ||
|
||
NYC.prototype._cacheDirectory = function () { | ||
return path.resolve(this.cwd, './', this.cacheDirectory) | ||
} | ||
|
||
NYC.prototype.mungeArgs = function (yargv) { | ||
var argv = process.argv.slice(1) | ||
argv = argv.slice(argv.indexOf(yargv._[0])) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -161,7 +161,7 @@ describe('nyc', function () { | |
}) | ||
|
||
describe('custom require hooks are installed', function () { | ||
it('wraps modules with coverage counters when the custom require hook compiles them', function () { | ||
/* it('wraps modules with coverage counters when the custom require hook compiles them', function () { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @novemberborn - we still have a few tests that test There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yea I need to go back and see what's still needed… |
||
var hook = sinon.spy(function (module, filename) { | ||
module._compile(fs.readFileSync(filename, 'utf8')) | ||
}) | ||
|
@@ -185,7 +185,7 @@ describe('nyc', function () { | |
|
||
// and the hook should have been called | ||
hook.calledOnce.should.be.true | ||
}) | ||
}) */ | ||
}) | ||
|
||
function testSignal (signal, done) { | ||
|
@@ -360,6 +360,8 @@ describe('nyc', function () { | |
}) | ||
nyc.wrap() | ||
|
||
nyc.instrumenter() | ||
|
||
istanbul.config.loadFile.calledWithMatch('.istanbul.yml').should.equal(true) | ||
istanbul.Instrumenter.calledWith({ | ||
coverageVariable: '__coverage__', | ||
|
@@ -379,6 +381,8 @@ describe('nyc', function () { | |
}) | ||
nyc.wrap() | ||
|
||
nyc.instrumenter() | ||
|
||
istanbul.config.loadFile.calledWithMatch(path.join('test', 'fixtures', '.istanbul.yml')).should.equal(true) | ||
istanbul.Instrumenter.calledWith({ | ||
coverageVariable: '__coverage__', | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could this be put in the home directory instead? It's annoying having to .gitignore this in every repo using nyc. ESLint does this by default: https://github.com/eslint/eslint/blob/master/docs/user-guide/command-line-interface.md#--cache-location
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It looks to me like ESLint's default behavior is similar to this one.
I agree, it is annoying to have to ignore it every time. I am not sure defaulting to the home directory is the best choice though. One advantage of the annoying folder in your project's base directory is that you notice it, and it becomes obvious what it is for. Hiding it away in your home directory puts it out of mind when you are trying to troubleshoot problems. Maybe we could put it in
./node_modules/nyc/.nyc_cache
, that way they end up clearing the cache if they rimrafnode_modules
while troubleshooting.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Or better yet, maybe we should put it in:
./node_modules/.cache/nyc
, and encourage other modules to use./node_modules/.cache/<module-name>
as well. Thentrash node_modules/.cache
becomes the de facto way to clear caches when troubleshooting.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That could work too.