From bdd246dca00968f81f5b14325748b84067641980 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Thu, 17 Dec 2020 14:10:24 -0800 Subject: [PATCH 1/6] ensure separate asset compilation states in subbuilds --- package.json | 2 +- scripts/build.js | 5 +++-- src/index.js | 32 +++++++++++++++++++++++--------- test/index.test.js | 6 +++--- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index bfb4920c..2c2bd248 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@sentry/node": "^4.3.0", "@slack/web-api": "^5.13.0", "@tensorflow/tfjs-node": "^0.3.0", - "@vercel/webpack-asset-relocator-loader": "0.9.1", + "@vercel/webpack-asset-relocator-loader": "github:vercel/webpack-asset-relocator-loader#subbuild-compilations", "analytics-node": "^3.3.0", "apollo-server-express": "^2.2.2", "arg": "^4.1.0", diff --git a/scripts/build.js b/scripts/build.js index c18895ab..5ee2fae3 100644 --- a/scripts/build.js +++ b/scripts/build.js @@ -54,8 +54,9 @@ async function main() { { filename: "ts-loader.js", minify, - v8cache: true - } + v8cache: true, + noAssetBuilds: true + }, ); checkUnknownAssets('ts-loader', Object.keys(tsLoaderAssets).filter(asset => !asset.startsWith('lib/') && !asset.startsWith('typescript/lib'))); diff --git a/src/index.js b/src/index.js index a1cba136..2051a009 100644 --- a/src/index.js +++ b/src/index.js @@ -135,10 +135,15 @@ function ncc ( let watcher, watchHandler, rebuildHandler; + const compilationStack = []; + var plugins = [ { apply(compiler) { - compiler.hooks.compilation.tap("relocate-loader", compilation => relocateLoader.initAssetCache(compilation)); + compiler.hooks.compilation.tap("relocate-loader", compilation => { + compilationStack.push(compilation); + relocateLoader.initAssetCache(compilation); + }); compiler.hooks.watchRun.tap("ncc", () => { if (rebuildHandler) rebuildHandler(); @@ -289,7 +294,10 @@ function ncc ( }); }); }) - .then(finalizeHandler); + .then(finalizeHandler, function (err) { + compilationStack.pop(); + throw err; + }); } else { if (typeof watch === 'object') { @@ -300,10 +308,14 @@ function ncc ( } let cachedResult; watcher = compiler.watch({}, async (err, stats) => { - if (err) + if (err) { + compilationStack.pop(); return watchHandler({ err }); - if (stats.hasErrors()) + } + if (stats.hasErrors()) { + compilationStack.pop(); return watchHandler({ err: stats.toString() }); + } const returnValue = await finalizeHandler(stats); if (watchHandler) watchHandler(returnValue); @@ -434,13 +446,13 @@ function ncc ( // for each .js / .mjs / .cjs file in the asset list, build that file with ncc itself if (!noAssetBuilds) { - let assetNames = Object.keys(assets); - assetNames.push(`${filename}${ext === '.cjs' ? '.js' : ''}`); + const compilation = compilationStack[compilationStack.length - 1]; + const assetNames = Object.keys(assets); for (const asset of assetNames) { if (!asset.endsWith('.js') && !asset.endsWith('.cjs') && !asset.endsWith('.ts') && !asset.endsWith('.mjs') || - asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs')) + asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs') || asset.endsWith('.d.ts')) continue; - const assetMeta = relocateLoader.getAssetMeta(asset); + const assetMeta = relocateLoader.getAssetMeta(asset, compilation); if (!assetMeta) continue; const path = assetMeta.path; @@ -458,7 +470,7 @@ function ncc ( noAssetBuilds: true, v8cache, filterAssetBase, - existingAssetNames: assetNames, + existingAssetNames: [...assetNames, `${filename}${ext === '.cjs' ? '.js' : ''}`], quiet, debugLog, transpileOnly, @@ -473,6 +485,8 @@ function ncc ( } } + compilationStack.pop(); + return { code, map: map ? JSON.stringify(map) : undefined, assets, symlinks, stats }; } } diff --git a/test/index.test.js b/test/index.test.js index 55ad1304..96812b22 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -77,7 +77,7 @@ for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) { }); } for (const cliTest of eval(fs.readFileSync(__dirname + "/cli.js").toString())) { - it(`should execute "ncc ${(cliTest.args || []).join(" ")}"`, async () => { + it.skip(`should execute "ncc ${(cliTest.args || []).join(" ")}"`, async () => { const ps = fork(__dirname + (coverage ? "/../src/cli.js" : "/../dist/ncc/cli.js"), cliTest.args || [], { stdio: "pipe" }); @@ -134,7 +134,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { // disabled pending https://github.com/zeit/ncc/issues/141 if (integrationTest.endsWith('loopback.js')) continue; - it(`should execute "ncc run ${integrationTest}"`, async () => { + it.skip(`should execute "ncc run ${integrationTest}"`, async () => { let expectedStdout; try { expectedStdout = fs.readFileSync(`${__dirname}/integration/${integrationTest}.stdout`).toString(); @@ -165,7 +165,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { }); } -it(`should execute "ncc build web-vitals" with target config`, async () => { +it.skip(`should execute "ncc build web-vitals" with target config`, async () => { if (global.gc) global.gc(); const stdout = new StoreStream(); const stderr = new StoreStream(); From ea854486e76a3da435e634fe00a9fccf413dffe9 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Thu, 17 Dec 2020 14:26:58 -0800 Subject: [PATCH 2/6] use master webpack-asset-relocator-loader --- package.json | 2 +- yarn.lock | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 2c2bd248..30ce9f04 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@sentry/node": "^4.3.0", "@slack/web-api": "^5.13.0", "@tensorflow/tfjs-node": "^0.3.0", - "@vercel/webpack-asset-relocator-loader": "github:vercel/webpack-asset-relocator-loader#subbuild-compilations", + "@vercel/webpack-asset-relocator-loader": "github:vercel/webpack-asset-relocator-loader#master", "analytics-node": "^3.3.0", "apollo-server-express": "^2.2.2", "arg": "^4.1.0", diff --git a/yarn.lock b/yarn.lock index 5826266a..c5e656f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1902,10 +1902,9 @@ dependencies: "@types/yargs-parser" "*" -"@vercel/webpack-asset-relocator-loader@0.9.1": +"@vercel/webpack-asset-relocator-loader@github:vercel/webpack-asset-relocator-loader#master": version "0.9.1" - resolved "https://registry.yarnpkg.com/@vercel/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-0.9.1.tgz#b9cd56e7b7de002830410aca708b76eea930a2cd" - integrity sha512-TJXZZa94a3mDB0jdHCDFaWzrFGwsNqpYNqqKeUieznpToN//9XDjQgwkgpvAhG04wwGBFIPvBBfZZdBsvGhLqQ== + resolved "https://codeload.github.com/vercel/webpack-asset-relocator-loader/tar.gz/be1bb7978e889638576ffd5ce229491c6b30c0ee" "@webassemblyjs/ast@1.9.0": version "1.9.0" From 74bde5f3d4cb631890ba7d49e3ba6f46fdec7b76 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Thu, 17 Dec 2020 14:35:53 -0800 Subject: [PATCH 3/6] reenable tests --- test/index.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/index.test.js b/test/index.test.js index 96812b22..55ad1304 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -77,7 +77,7 @@ for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) { }); } for (const cliTest of eval(fs.readFileSync(__dirname + "/cli.js").toString())) { - it.skip(`should execute "ncc ${(cliTest.args || []).join(" ")}"`, async () => { + it(`should execute "ncc ${(cliTest.args || []).join(" ")}"`, async () => { const ps = fork(__dirname + (coverage ? "/../src/cli.js" : "/../dist/ncc/cli.js"), cliTest.args || [], { stdio: "pipe" }); @@ -134,7 +134,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { // disabled pending https://github.com/zeit/ncc/issues/141 if (integrationTest.endsWith('loopback.js')) continue; - it.skip(`should execute "ncc run ${integrationTest}"`, async () => { + it(`should execute "ncc run ${integrationTest}"`, async () => { let expectedStdout; try { expectedStdout = fs.readFileSync(`${__dirname}/integration/${integrationTest}.stdout`).toString(); @@ -165,7 +165,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) { }); } -it.skip(`should execute "ncc build web-vitals" with target config`, async () => { +it(`should execute "ncc build web-vitals" with target config`, async () => { if (global.gc) global.gc(); const stdout = new StoreStream(); const stderr = new StoreStream(); From c30689745cfc0ca5ab813da214c8168a37a83fbf Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Thu, 17 Dec 2020 14:38:35 -0800 Subject: [PATCH 4/6] fixup asset deduping --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 2051a009..0825b1c2 100644 --- a/src/index.js +++ b/src/index.js @@ -470,7 +470,7 @@ function ncc ( noAssetBuilds: true, v8cache, filterAssetBase, - existingAssetNames: [...assetNames, `${filename}${ext === '.cjs' ? '.js' : ''}`], + existingAssetNames: [...Object.keys(assets), `${filename}${ext === '.cjs' ? '.js' : ''}`], quiet, debugLog, transpileOnly, From 21b57c2e83ba7d053d18cac9dae798b492f25740 Mon Sep 17 00:00:00 2001 From: Guy Bedford Date: Thu, 17 Dec 2020 15:37:10 -0800 Subject: [PATCH 5/6] better asset deduping --- src/index.js | 70 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/src/index.js b/src/index.js index 0825b1c2..e1847216 100644 --- a/src/index.js +++ b/src/index.js @@ -447,41 +447,53 @@ function ncc ( // for each .js / .mjs / .cjs file in the asset list, build that file with ncc itself if (!noAssetBuilds) { const compilation = compilationStack[compilationStack.length - 1]; - const assetNames = Object.keys(assets); - for (const asset of assetNames) { + let existingAssetNames = Object.keys(assets); + existingAssetNames.push(`${filename}${ext === '.cjs' ? '.js' : ''}`); + const subbuildAssets = []; + for (const asset of Object.keys(assets)) { if (!asset.endsWith('.js') && !asset.endsWith('.cjs') && !asset.endsWith('.ts') && !asset.endsWith('.mjs') || - asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs') || asset.endsWith('.d.ts')) + asset.endsWith('.cache.js') || asset.endsWith('.cache.cjs') || asset.endsWith('.cache.ts') || asset.endsWith('.cache.mjs') || asset.endsWith('.d.ts')) { + existingAssetNames.push(asset); continue; + } const assetMeta = relocateLoader.getAssetMeta(asset, compilation); - if (!assetMeta) + if (!assetMeta || !assetMeta.path) { + existingAssetNames.push(asset); continue; + } + subbuildAssets.push(asset); + } + for (const asset of subbuildAssets) { + const assetMeta = relocateLoader.getAssetMeta(asset, compilation); const path = assetMeta.path; - if (path) { - const { code, assets: subbuildAssets, symlinks: subbuildSymlinks, stats: subbuildStats } = await ncc(path, { - cache, - externals, - filename: asset, - minify, - sourceMap, - sourceMapRegister, - sourceMapBasePrefix, - // dont recursively asset build - // could be supported with seen tracking - noAssetBuilds: true, - v8cache, - filterAssetBase, - existingAssetNames: [...Object.keys(assets), `${filename}${ext === '.cjs' ? '.js' : ''}`], - quiet, - debugLog, - transpileOnly, - license, - target - }); - Object.assign(symlinks, subbuildSymlinks); - Object.assign(stats, subbuildStats); - assets[asset] = { source: code, permissions: assetMeta.permissions }; - Object.assign(assets, subbuildAssets); + const { code, assets: subbuildAssets, symlinks: subbuildSymlinks, stats: subbuildStats } = await ncc(path, { + cache, + externals, + filename: asset, + minify, + sourceMap, + sourceMapRegister, + sourceMapBasePrefix, + // dont recursively asset build + // could be supported with seen tracking + noAssetBuilds: true, + v8cache, + filterAssetBase, + existingAssetNames, + quiet, + debugLog, + transpileOnly, + license, + target + }); + Object.assign(symlinks, subbuildSymlinks); + Object.assign(stats, subbuildStats); + for (const subasset of Object.keys(subbuildAssets)) { + assets[subasset] = subbuildAssets[subasset]; + if (!existingAssetNames.includes(subasset)) + existingAssetNames.push(subasset); } + assets[asset] = { source: code, permissions: assetMeta.permissions }; } } From 4a3bcb425dda57ce24e3f90468342c267f14a76d Mon Sep 17 00:00:00 2001 From: Joe Haddad Date: Fri, 18 Dec 2020 14:40:09 -0500 Subject: [PATCH 6/6] chore: upgrade to 1.0.0 --- package.json | 2 +- yarn.lock | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 30ce9f04..d4555b51 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@sentry/node": "^4.3.0", "@slack/web-api": "^5.13.0", "@tensorflow/tfjs-node": "^0.3.0", - "@vercel/webpack-asset-relocator-loader": "github:vercel/webpack-asset-relocator-loader#master", + "@vercel/webpack-asset-relocator-loader": "1.0.0", "analytics-node": "^3.3.0", "apollo-server-express": "^2.2.2", "arg": "^4.1.0", diff --git a/yarn.lock b/yarn.lock index c5e656f3..590209ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1902,9 +1902,10 @@ dependencies: "@types/yargs-parser" "*" -"@vercel/webpack-asset-relocator-loader@github:vercel/webpack-asset-relocator-loader#master": - version "0.9.1" - resolved "https://codeload.github.com/vercel/webpack-asset-relocator-loader/tar.gz/be1bb7978e889638576ffd5ce229491c6b30c0ee" +"@vercel/webpack-asset-relocator-loader@1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@vercel/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-1.0.0.tgz#cc3cae3b3cc8f3f631552e1ca97e566f597d5e5b" + integrity sha512-JZGVEFBOR0I7ccwewsWSSOB/ke6wYV4e09qQHtTBLTe/zgWrsjes7SX1Xt9M1UgiMTkhZ/0jyJpdxEqrpaYRMA== "@webassemblyjs/ast@1.9.0": version "1.9.0"