From 8c1c97c5479f5e00f2f473e915138b02fbeb0e7a Mon Sep 17 00:00:00 2001 From: Daniel Lando Date: Mon, 17 Jan 2022 18:02:15 +0100 Subject: [PATCH 1/3] fix: copyFile and copyFileSync patch Fixes #420 --- prelude/bootstrap.js | 66 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index 7285b7c86..c50f1b472 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -158,7 +158,7 @@ function createMountpoint(interior, exterior) { mountpoints.push({ interior, exterior }); } -function copyFileSync(source, target) { +function customCopyFile(source, target) { let targetFile = target; // If target is a directory, a new file with the same name will be created @@ -188,7 +188,7 @@ function copyFolderRecursiveSync(source, target) { if (fs.lstatSync(curSource).isDirectory()) { copyFolderRecursiveSync(curSource, targetFolder); } else { - copyFileSync(curSource, targetFolder); + customCopyFile(curSource, targetFolder); } }); } @@ -572,6 +572,8 @@ function payloadFileSync(pointer) { mkdirSync: fs.mkdirSync, mkdir: fs.mkdir, createReadStream: fs.createReadStream, + copyFileSync: fs.copyFileSync, + copyFile: fs.copyFile, }; ancestor.realpathSync.native = fs.realpathSync; @@ -1066,6 +1068,64 @@ function payloadFileSync(pointer) { }); }; + fs.copyFile = function copyFile(src, dest, flags, callback) { + if (!insideSnapshot(path.resolve(src))) { + ancestor.copyFile(src, dest, flags, callback); + return; + } + if (typeof flags === 'function') { + callback = flags; + flags = 0; + } else if (typeof callback !== 'function') { + throw new TypeError('Callback must be a function'); + } + + fs.readFile(src, (readError, content) => { + if (readError) { + callback(readError); + return; + } + if (flags & fs.constants.COPYFILE_EXCL) { + fs.stat(dest, (statError) => { + if (!statError) { + callback( + Object.assign(new Error('File already exists'), { + code: 'EEXIST', + }) + ); + return; + } + if (statError.code !== 'ENOENT') { + callback(statError); + return; + } + fs.writeFile(dest, content, callback); + }); + } else { + fs.writeFile(dest, content, callback); + } + }); + }; + + fs.copyFileSync = function copyFileSync(src, dest, flags) { + if (!insideSnapshot(path.resolve(src))) { + ancestor.copyFileSync(src, dest, flags); + return; + } + const content = fs.readFileSync(src); + if (flags & fs.constants.COPYFILE_EXCL) { + try { + fs.statSync(dest); + } catch (statError) { + if (statError.code !== 'ENOENT') throw statError; + fs.writeFileSync(dest, content); + return; + } + throw Object.assign(new Error('File already exists'), { code: 'EEXIST' }); + } + fs.writeFileSync(dest, content); + }; + // /////////////////////////////////////////////////////////////// // writeFile ///////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// @@ -1540,6 +1600,7 @@ function payloadFileSync(pointer) { lstat: fs.promises.lstat, fstat: fs.promises.fstat, access: fs.promises.access, + copyFile: fs.promises.copyFile, }; fs.promises.open = async function open(path_) { @@ -1586,6 +1647,7 @@ function payloadFileSync(pointer) { // this one use promisify on purpose fs.promises.readdir = util.promisify(fs.readdir); + fs.promises.copyFile = util.promisify(fs.copyFile); /* fs.promises.read = util.promisify(fs.read); From 9fc04abeadad6e036ca7f66b5a9674351d0bee31 Mon Sep 17 00:00:00 2001 From: Nikolai Kolodziej <7687617+kldzj@users.noreply.github.com> Date: Thu, 3 Feb 2022 03:44:52 +0100 Subject: [PATCH 2/3] Copy in streams/chunks (#1484) (#1499) --- prelude/bootstrap.js | 84 ++++++++++++------- test/test-420-copy-from-snapshot/.gitignore | 3 + test/test-420-copy-from-snapshot/copy.js | 19 +++++ .../input/test.json | 3 + test/test-420-copy-from-snapshot/main.js | 30 +++++++ test/test-420-copy-from-snapshot/package.json | 8 ++ 6 files changed, 117 insertions(+), 30 deletions(-) create mode 100644 test/test-420-copy-from-snapshot/.gitignore create mode 100644 test/test-420-copy-from-snapshot/copy.js create mode 100644 test/test-420-copy-from-snapshot/input/test.json create mode 100644 test/test-420-copy-from-snapshot/main.js create mode 100644 test/test-420-copy-from-snapshot/package.json diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index c50f1b472..d0ea831e7 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -158,7 +158,28 @@ function createMountpoint(interior, exterior) { mountpoints.push({ interior, exterior }); } -function customCopyFile(source, target) { +const DEFAULT_COPY_CHUNK_SIZE = 10 * 1024 * 1024; // 10 MB +function copyInChunks( + source, + target, + chunkSize = DEFAULT_COPY_CHUNK_SIZE, + fs_ = fs +) { + const sourceFile = fs_.openSync(source, 'r'); + const targetFile = fs_.openSync(target, 'w'); + + let bytesRead = 1; + while (bytesRead > 0) { + const buffer = Buffer.alloc(chunkSize); + bytesRead = fs_.readSync(sourceFile, buffer, 0, chunkSize); + fs_.writeSync(targetFile, buffer, 0, bytesRead); + } + + fs_.closeSync(sourceFile); + fs_.closeSync(targetFile); +} + +function customCopyFile(source, target, chunkSize) { let targetFile = target; // If target is a directory, a new file with the same name will be created @@ -168,7 +189,7 @@ function customCopyFile(source, target) { } } - fs.writeFileSync(targetFile, fs.readFileSync(source)); + copyInChunks(source, targetFile, chunkSize); } function copyFolderRecursiveSync(source, target) { @@ -1080,31 +1101,33 @@ function payloadFileSync(pointer) { throw new TypeError('Callback must be a function'); } - fs.readFile(src, (readError, content) => { - if (readError) { - callback(readError); - return; - } - if (flags & fs.constants.COPYFILE_EXCL) { - fs.stat(dest, (statError) => { - if (!statError) { - callback( - Object.assign(new Error('File already exists'), { - code: 'EEXIST', - }) - ); - return; - } - if (statError.code !== 'ENOENT') { - callback(statError); - return; - } - fs.writeFile(dest, content, callback); - }); - } else { - fs.writeFile(dest, content, callback); - } - }); + function _streamCopy() { + fs.createReadStream(src) + .on('error', callback) + .pipe(fs.createWriteStream(dest)) + .on('error', callback) + .on('finish', callback); + } + + if (flags & fs.constants.COPYFILE_EXCL) { + fs.stat(dest, (statError) => { + if (!statError) { + callback( + Object.assign(new Error('File already exists'), { + code: 'EEXIST', + }) + ); + return; + } + if (statError.code !== 'ENOENT') { + callback(statError); + return; + } + _streamCopy(); + }); + } else { + _streamCopy(); + } }; fs.copyFileSync = function copyFileSync(src, dest, flags) { @@ -1112,18 +1135,19 @@ function payloadFileSync(pointer) { ancestor.copyFileSync(src, dest, flags); return; } - const content = fs.readFileSync(src); + if (flags & fs.constants.COPYFILE_EXCL) { try { fs.statSync(dest); } catch (statError) { if (statError.code !== 'ENOENT') throw statError; - fs.writeFileSync(dest, content); + copyInChunks(src, dest, DEFAULT_COPY_CHUNK_SIZE, fs); return; } + throw Object.assign(new Error('File already exists'), { code: 'EEXIST' }); } - fs.writeFileSync(dest, content); + copyInChunks(src, dest, DEFAULT_COPY_CHUNK_SIZE, fs); }; // /////////////////////////////////////////////////////////////// diff --git a/test/test-420-copy-from-snapshot/.gitignore b/test/test-420-copy-from-snapshot/.gitignore new file mode 100644 index 000000000..b7482ccf8 --- /dev/null +++ b/test/test-420-copy-from-snapshot/.gitignore @@ -0,0 +1,3 @@ +output +sync.json +async.json diff --git a/test/test-420-copy-from-snapshot/copy.js b/test/test-420-copy-from-snapshot/copy.js new file mode 100644 index 000000000..0a4f23e6a --- /dev/null +++ b/test/test-420-copy-from-snapshot/copy.js @@ -0,0 +1,19 @@ +#!/usr/bin/env node + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const testPath = path.resolve(__dirname, 'input/test.json'); +console.log(fs.readFileSync(testPath, 'utf8')); + +const syncPath = path.resolve(process.cwd(), 'output/sync.json'); +fs.copyFileSync(testPath, syncPath); +console.log(fs.readFileSync(syncPath, 'utf8')); + +const asyncPath = path.resolve(process.cwd(), 'output/async.json'); +fs.copyFile(testPath, asyncPath, (err) => { + if (err) throw err; + console.log(fs.readFileSync(asyncPath, 'utf8')); +}); diff --git a/test/test-420-copy-from-snapshot/input/test.json b/test/test-420-copy-from-snapshot/input/test.json new file mode 100644 index 000000000..7a9e86441 --- /dev/null +++ b/test/test-420-copy-from-snapshot/input/test.json @@ -0,0 +1,3 @@ +{ + "key": "value" +} diff --git a/test/test-420-copy-from-snapshot/main.js b/test/test-420-copy-from-snapshot/main.js new file mode 100644 index 000000000..0a04ea41f --- /dev/null +++ b/test/test-420-copy-from-snapshot/main.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node + +'use strict'; + +const path = require('path'); +const assert = require('assert'); +const utils = require('../utils.js'); + +assert(!module.parent); +assert(__dirname === process.cwd()); + +const target = process.argv[2] || 'host'; +const input = './copy.js'; +const output = './output/test-output.exe'; + +utils.mkdirp.sync(path.dirname(output)); +utils.pkg.sync(['--target', target, '--output', output, '.']); + +let left, right; +left = utils.spawn.sync('node', [path.basename(input)], { + cwd: path.dirname(input), +}); + +right = utils.spawn.sync(output, [], { + cwd: path.dirname(input), +}); + +assert.strictEqual(left, right); +utils.vacuum.sync(path.dirname(output)); +utils.vacuum.sync(path.join(__dirname, '/*sync.json')); diff --git a/test/test-420-copy-from-snapshot/package.json b/test/test-420-copy-from-snapshot/package.json new file mode 100644 index 000000000..8ce379bda --- /dev/null +++ b/test/test-420-copy-from-snapshot/package.json @@ -0,0 +1,8 @@ +{ + "bin": "copy.js", + "pkg": { + "assets": [ + "input/**/*" + ] + } +} From ed293440642e004111e6fcdceafcbbc153f72bde Mon Sep 17 00:00:00 2001 From: Jesse Chan Date: Wed, 2 Feb 2022 18:57:56 -0800 Subject: [PATCH 3/3] bootstrap: drop unneeded "customCopyFile" --- prelude/bootstrap.js | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/prelude/bootstrap.js b/prelude/bootstrap.js index d0ea831e7..df34da631 100644 --- a/prelude/bootstrap.js +++ b/prelude/bootstrap.js @@ -179,19 +179,7 @@ function copyInChunks( fs_.closeSync(targetFile); } -function customCopyFile(source, target, chunkSize) { - let targetFile = target; - - // If target is a directory, a new file with the same name will be created - if (fs.existsSync(target)) { - if (fs.lstatSync(target).isDirectory()) { - targetFile = path.join(target, path.basename(source)); - } - } - - copyInChunks(source, targetFile, chunkSize); -} - +// TODO: replace this with fs.cpSync when we drop Node < 16 function copyFolderRecursiveSync(source, target) { let files = []; @@ -209,7 +197,10 @@ function copyFolderRecursiveSync(source, target) { if (fs.lstatSync(curSource).isDirectory()) { copyFolderRecursiveSync(curSource, targetFolder); } else { - customCopyFile(curSource, targetFolder); + fs.copyFileSync( + curSource, + path.join(targetFolder, path.basename(curSource)) + ); } }); }