Skip to content

Commit

Permalink
fix: stringify stats using streaming approach
Browse files Browse the repository at this point in the history
Change the approach for stats stringify to use streams instead of JSON.stringify() + sync write. No API or functionality is changed.

It makes stats available for huge projects. Changes has the follow effects:

Memory
Allows to avoid max string length limit of V8 engine (500MB). In other words, stats becomes available for huge projects, whose stats exceed 500MB.
Reduces memory consumption on serialization.
Adds time penalty. However, this make sense for huge stats only and seems to be an acceptable, given that otherwise stats may be unavailable (due max string limit hit or out of memory).
Adds new dependency "json-ext", which is dependency free and most effective solution for a huge JSON parse/stringify at the moment, and other solutions have troubles to handle huge JSON (see https://github.com/discoveryjs/json-ext/blob/master/benchmarks/README.md#stream-stringifying).
  • Loading branch information
lahmatiy committed Dec 5, 2020
1 parent 3e5f946 commit 9b5d04e
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 13 deletions.
29 changes: 16 additions & 13 deletions packages/webpack-cli/lib/webpack-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ const path = require('path');
const packageExists = require('./utils/package-exists');
const webpack = packageExists('webpack') ? require('webpack') : undefined;
const webpackMerge = require('webpack-merge');
const { writeFileSync, existsSync } = require('fs');
const { createWriteStream, existsSync } = require('fs');
const { options: coloretteOptions, yellow } = require('colorette');
const { stringifyStream: createJsonStringifyStream } = require('@discoveryjs/json-ext');

const logger = require('./utils/logger');
const { cli, flags } = require('./utils/cli-flags');
Expand Down Expand Up @@ -485,21 +486,23 @@ class WebpackCLI {
const foundStats = compiler.compilers
? { children: compiler.compilers.map(getStatsOptionsFromCompiler) }
: getStatsOptionsFromCompiler(compiler);
const handleWriteError = (error) => {
logger.error(error);
process.exit(2);
};

if (args.json === true) {
process.stdout.write(JSON.stringify(stats.toJson(foundStats)) + '\n');
createJsonStringifyStream(stats.toJson(foundStats))
.on('error', handleWriteError)
.pipe(process.stdout)
.on('error', handleWriteError)
.on('close', () => process.stdout.write('\n'));
} else if (typeof args.json === 'string') {
const JSONStats = JSON.stringify(stats.toJson(foundStats));

try {
writeFileSync(args.json, JSONStats);

logger.success(`stats are successfully stored as json to ${args.json}`);
} catch (error) {
logger.error(error);

process.exit(2);
}
createJsonStringifyStream(stats.toJson(foundStats))
.on('error', handleWriteError)
.pipe(createWriteStream(args.json))
.on('error', handleWriteError)
.on('close', () => logger.success(`stats are successfully stored as json to ${args.json}`));
} else {
const printedStats = stats.toString(foundStats);

Expand Down
1 change: 1 addition & 0 deletions packages/webpack-cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"lib"
],
"dependencies": {
"@discoveryjs/json-ext": "^0.5.0",
"@webpack-cli/info": "^1.1.0",
"@webpack-cli/serve": "^1.1.0",
"colorette": "^1.2.1",
Expand Down

0 comments on commit 9b5d04e

Please sign in to comment.