Skip to content

Commit

Permalink
stream: Raise readable high water mark in powers of 2
Browse files Browse the repository at this point in the history
This prevents excessively raising the buffer level in tiny increments in
pathological cases.
  • Loading branch information
isaacs committed Mar 6, 2013
1 parent a978bed commit 9208c89
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 5 deletions.
19 changes: 17 additions & 2 deletions lib/_stream_readable.js
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,19 @@ Readable.prototype.setEncoding = function(enc) {
this._readableState.decoder = new StringDecoder(enc);
};

// Don't raise the hwm > 128MB
var MAX_HWM = 0x800000;
function roundUpToNextPowerOf2(n) {
if (n >= MAX_HWM) {
n = MAX_HWM;
} else {
// Get the next highest power of 2
n--;
for (var p = 1; p < 32; p <<= 1) n |= n >> p;
n++;
}
return n;
}

function howMuchToRead(n, state) {
if (state.length === 0 && state.ended)
Expand All @@ -181,9 +194,11 @@ function howMuchToRead(n, state) {
return 0;

// If we're asking for more than the target buffer level,
// then raise the water mark.
// then raise the water mark. Bump up to the next highest
// power of 2, to prevent increasing it excessively in tiny
// amounts.
if (n > state.highWaterMark)
state.highWaterMark = n;
state.highWaterMark = roundUpToNextPowerOf2(n);

// don't have that much. return null, unless we've ended.
if (n > state.length) {
Expand Down
13 changes: 10 additions & 3 deletions test/simple/test-stream-readable-flow-recursion.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,15 @@ process.throwDeprecation = true;

var stream = new Readable({ highWaterMark: 2 });
var reads = 0;
var total = 5000;
stream._read = function(size) {
reads++;
stream.push(new Buffer(size));
size = Math.min(size, total);
total -= size;
if (size === 0)
stream.push(null);
else
stream.push(new Buffer(size));
};

var depth = 0;
Expand All @@ -61,8 +67,9 @@ flow(stream, 5000, function() {
process.on('exit', function(code) {
assert.equal(reads, 2);
// we pushed up the high water mark
assert.equal(stream._readableState.highWaterMark, 5000);
assert.equal(stream._readableState.length, 5000);
assert.equal(stream._readableState.highWaterMark, 8192);
// length is 0 right now, because we pulled it all out.
assert.equal(stream._readableState.length, 0);
assert(!code);
assert.equal(depth, 0);
console.log('ok');
Expand Down

0 comments on commit 9208c89

Please sign in to comment.