From a62a62f3a4476518433ef686794bdc1e27754038 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Sat, 8 Jun 2024 18:09:17 -0700 Subject: [PATCH 01/51] fix #11681 and some other stuff --- src/cli.zig | 8 ++ src/install/install.zig | 82 +++++++++++----- src/install/lockfile.zig | 4 + test/cli/install/bun-install-patch.test.ts | 104 +++++++++++++++++++++ 4 files changed, 174 insertions(+), 24 deletions(-) diff --git a/src/cli.zig b/src/cli.zig index 6f7b39726ec022..af883346a78210 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -1005,6 +1005,8 @@ pub const HelpCommand = struct { \\ update {s:<16} Update outdated dependencies \\ link [\] Register or link a local npm package \\ unlink Unregister a local npm package + \\ patch \ Prepare a package for patching + \\ patch-commit \ Install a modified package \\ pm \ Additional package management utilities \\ \\ build ./a.ts ./b.jsx Bundle TypeScript & JavaScript into a single file @@ -2293,6 +2295,12 @@ pub const Command = struct { Output.pretty("Usage: bun completions", .{}); Output.flush(); }, + Command.Tag.PatchCommand => { + Install.PackageManager.CommandLineArguments.printHelp(.patch); + }, + Command.Tag.PatchCommitCommand => { + Install.PackageManager.CommandLineArguments.printHelp(.@"patch-commit"); + }, Command.Tag.ExecCommand => { Output.pretty( \\Usage: bun exec \ diff --git a/src/install/install.zig b/src/install/install.zig index 34c8ae93ff111b..4004f028939103 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -1087,7 +1087,6 @@ pub fn NewPackageInstall(comptime kind: PkgInstallKind) type { return strings.eqlLong(repo.resolved.slice(buf), bun_tag_file, true); } - // TODO: patched dependencies pub fn verify( this: *@This(), resolution: *const Resolution, @@ -2717,6 +2716,8 @@ pub const PackageManager = struct { // dependency name -> original version information updating_packages: bun.StringArrayHashMapUnmanaged(PackageUpdateInfo) = .{}, + patched_dependencies_to_remove: std.ArrayHashMapUnmanaged(PackageNameAndVersionHash, void, ArrayIdentityContext.U64, false) = .{}, + pub const PackageUpdateInfo = struct { original_version_literal: string, is_alias: bool, @@ -5569,7 +5570,6 @@ pub const PackageManager = struct { manager.network_resolve_batch = .{}; manager.patch_apply_batch = .{}; manager.patch_calc_hash_batch = .{}; - // TODO probably have to put patch tasks here return count; } @@ -7195,10 +7195,10 @@ pub const PackageManager = struct { if (subcommand == .patch) { // TODO args - } else if (subcommand == .patch_commit) { + } else if (subcommand == .@"patch-commit") { this.patch_features = .{ .commit = .{ - .patches_dir = cli.patch_commit.patches_dir, + .patches_dir = cli.@"patch-commit".patches_dir, }, }; } @@ -8145,7 +8145,7 @@ pub const PackageManager = struct { link, unlink, patch, - patch_commit, + @"patch-commit", }; pub fn init(ctx: Command.Context, comptime subcommand: Subcommand) !*PackageManager { @@ -8634,7 +8634,7 @@ pub const PackageManager = struct { } pub inline fn patchCommit(ctx: Command.Context) !void { - try updatePackageJSONAndInstallCatchError(ctx, .patch_commit); + try updatePackageJSONAndInstallCatchError(ctx, .@"patch-commit"); } pub inline fn update(ctx: Command.Context) !void { @@ -9068,7 +9068,7 @@ pub const PackageManager = struct { concurrent_scripts: ?usize = null, patch: PatchOpts = .{}, - patch_commit: PatchCommitOpts = .{}, + @"patch-commit": PatchCommitOpts = .{}, const PatchOpts = struct { edit_dir: ?[]const u8 = null, @@ -9163,7 +9163,7 @@ pub const PackageManager = struct { // Output.pretty("\n\n" ++ outro_text ++ "\n", .{}); Output.flush(); }, - Subcommand.patch_commit => { + Subcommand.@"patch-commit" => { const intro_text = \\Usage: bun patch-commit \ \\ @@ -9300,7 +9300,7 @@ pub const PackageManager = struct { .link => link_params, .unlink => unlink_params, .patch => patch_params, - .patch_commit => patch_commit_params, + .@"patch-commit" => patch_commit_params, }; var diag = clap.Diagnostic{}; @@ -9338,7 +9338,6 @@ pub const PackageManager = struct { // link and unlink default to not saving, all others default to // saving. - // TODO: I think `bun patch` command goes here if (comptime subcommand == .link or subcommand == .unlink) { cli.no_save = !args.flag("--save"); } else { @@ -9349,8 +9348,8 @@ pub const PackageManager = struct { cli.patch = .{}; } - if (comptime subcommand == .patch_commit) { - cli.patch_commit = .{ + if (comptime subcommand == .@"patch-commit") { + cli.@"patch-commit" = .{ .patches_dir = args.option("--patches-dir") orelse "patches", }; } @@ -9430,7 +9429,7 @@ pub const PackageManager = struct { Global.crash(); } - if (subcommand == .patch_commit and cli.positionals.len < 2) { + if (subcommand == .@"patch-commit" and cli.positionals.len < 2) { Output.errGeneric("Missing pkg folder to patch\n", .{}); Global.crash(); } @@ -9802,7 +9801,7 @@ pub const PackageManager = struct { } } }, - .patch_commit => { + .@"patch-commit" => { _ = manager.lockfile.loadFromDisk( manager, manager.allocator, @@ -10037,10 +10036,10 @@ pub const PackageManager = struct { fn preparePatch(manager: *PackageManager) !void { const @"pkg + maybe version to patch" = manager.options.positionals[1]; const name: []const u8, const version: ?[]const u8 = brk: { - if (std.mem.indexOfScalar(u8, @"pkg + maybe version to patch", '@')) |version_delimiter| { + if (std.mem.indexOfScalar(u8, @"pkg + maybe version to patch"[1..], '@')) |version_delimiter| { break :brk .{ @"pkg + maybe version to patch"[0..version_delimiter], - @"pkg + maybe version to patch"[version_delimiter + 1 ..], + @"pkg + maybe version to patch"[1..][version_delimiter + 1 ..], }; } break :brk .{ @@ -11056,14 +11055,29 @@ pub const PackageManager = struct { break :brk ""; } else std.fmt.bufPrint(&resolution_buf, "{}", .{resolution.fmt(buf, .posix)}) catch unreachable; - const patch_patch, const patch_contents_hash, const patch_name_and_version_hash = brk: { - if (this.manager.lockfile.patched_dependencies.entries.len == 0) break :brk .{ null, null, null }; + if (std.mem.eql(u8, "is-even", name)) { + std.debug.print("HELLO!\n", .{}); + } + + const patch_patch, const patch_contents_hash, const patch_name_and_version_hash, const remove_patch = brk: { + if (this.manager.lockfile.patched_dependencies.entries.len == 0 and this.manager.patched_dependencies_to_remove.entries.len) break :brk .{ null, null, null, false }; var sfb = std.heap.stackFallback(1024, this.lockfile.allocator); const name_and_version = std.fmt.allocPrint(sfb.get(), "{s}@{s}", .{ name, package_version }) catch unreachable; defer sfb.get().free(name_and_version); const name_and_version_hash = String.Builder.stringHash(name_and_version); - const patchdep = this.lockfile.patched_dependencies.get(name_and_version_hash) orelse break :brk .{ null, null, null }; + const patchdep = this.lockfile.patched_dependencies.get(name_and_version_hash) orelse { + const to_remove = this.manager.patched_dependencies_to_remove.contains(name_and_version_hash); + if (to_remove) { + break :brk .{ + null, + null, + name_and_version_hash, + true, + }; + } + break :brk .{ null, null, null, false }; + }; bun.assert(!patchdep.patchfile_hash_is_null); // if (!patchdep.patchfile_hash_is_null) { // this.manager.enqueuePatchTask(PatchTask.newCalcPatchHash(this, package_id, name_and_version_hash, dependency_id, url: string)) @@ -11072,6 +11086,7 @@ pub const PackageManager = struct { patchdep.path.slice(this.lockfile.buffers.string_bytes.items), patchdep.patchfileHash().?, name_and_version_hash, + false, }; }; @@ -11200,7 +11215,7 @@ pub const PackageManager = struct { }, } - const needs_install = this.force_install or this.skip_verify_installed_version_number or !needs_verify or !installer.verify( + const needs_install = this.force_install or this.skip_verify_installed_version_number or !needs_verify or remove_patch or !installer.verify( resolution, buf, this.root_node_modules_folder, @@ -11208,7 +11223,7 @@ pub const PackageManager = struct { this.summary.skipped += @intFromBool(!needs_install); if (needs_install) { - if (resolution.tag.canEnqueueInstallTask() and installer.packageMissingFromCache(this.manager, package_id)) { + if (!remove_patch and resolution.tag.canEnqueueInstallTask() and installer.packageMissingFromCache(this.manager, package_id)) { if (comptime Environment.allow_assert) { bun.assert(resolution.canEnqueueInstallTask()); } @@ -11292,7 +11307,7 @@ pub const PackageManager = struct { // above checks if unpatched package is in cache, if not null apply patch in temp directory, copy // into cache, then install into node_modules if (!installer.patch.isNull()) { - if (installer.patchedPackageMissingFromCache(this.manager, package_id, installer.patch.patch_contents_hash)) { + if (installer.patchedPackageMissingFromCache(this.manager, package_id)) { const task = PatchTask.newApplyPatchHash( this.manager, package_id, @@ -11337,7 +11352,7 @@ pub const PackageManager = struct { const install_result = switch (resolution.tag) { .symlink, .workspace => installer.installFromLink(this.skip_delete, destination_dir), - else => installer.install(this.skip_delete, destination_dir), + else => installer.install(this.skip_delete and (!remove_patch), destination_dir), }; switch (install_result) { @@ -12568,7 +12583,6 @@ pub const PackageManager = struct { // Update patched dependencies { var iter = lockfile.patched_dependencies.iterator(); - // TODO: if one key is present in manager.lockfile and not present in lockfile we should get rid of it while (iter.next()) |entry| { const pkg_name_and_version_hash = entry.key_ptr.*; bun.debugAssert(entry.value_ptr.patchfile_hash_is_null); @@ -12587,6 +12601,26 @@ pub const PackageManager = struct { gop.value_ptr.setPatchfileHash(null); } } + + var count: usize = 0; + iter = manager.lockfile.patched_dependencies.iterator(); + while (iter.next()) |entry| { + if (!lockfile.patched_dependencies.contains(entry.key_ptr.*)) { + count += 1; + } + } + if (count > 0) { + try manager.patched_dependencies_to_remove.ensureTotalCapacity(manager.allocator, count); + iter = manager.lockfile.patched_dependencies.iterator(); + while (iter.next()) |entry| { + if (!lockfile.patched_dependencies.contains(entry.key_ptr.*)) { + try manager.patched_dependencies_to_remove.put(manager.allocator, entry.key_ptr.*, {}); + } + } + for (manager.patched_dependencies_to_remove.keys()) |hash| { + _ = manager.lockfile.patched_dependencies.orderedRemove(hash); + } + } } builder.clamp(); diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index 4a48b3e03d6e01..bc93559701d0ee 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -3775,6 +3775,10 @@ pub const Package = extern struct { )) break :patched_dependencies_changed true; } else break :patched_dependencies_changed true; } + iter = from_lockfile.patched_dependencies.iterator(); + while (iter.next()) |entry| { + if (!to_lockfile.patched_dependencies.contains(entry.key_ptr.*)) break :patched_dependencies_changed true; + } break :patched_dependencies_changed false; }; diff --git a/test/cli/install/bun-install-patch.test.ts b/test/cli/install/bun-install-patch.test.ts index da972c13b73b0c..398e5929b6def4 100644 --- a/test/cli/install/bun-install-patch.test.ts +++ b/test/cli/install/bun-install-patch.test.ts @@ -351,6 +351,58 @@ index c8950c17b265104bcf27f8c345df1a1b13a78950..7ce57ab96400ab0ff4fac7e06f6e02c2 } }); + describe("should work when patches are removed", async () => { + for (const [version, patchVersion_] of versions) { + const patchFilename = filepathEscape(`is-even@${version}.patch`); + const patchVersion = patchVersion_ ?? version; + test(version, async () => { + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "patchedDependencies": { + [`is-even@${patchVersion}`]: `patches/${patchFilename}`, + }, + "dependencies": { + "is-even": version, + }, + }), + patches: { + [patchFilename]: is_even_patch2, + }, + "index.ts": /* ts */ `import isEven from 'is-even'; isEven(2); console.log('lol')`, + }); + + console.log("FILEDIR", filedir); + + await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + throw new Error("woopsie!"); + + await $`echo ${JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "patchedDependencies": { + [`is-odd@0.1.2`]: `patches/is-odd@0.1.2.patch`, + }, + "dependencies": { + "is-even": version, + }, + })} > package.json` + .env(bunEnv) + .cwd(filedir); + + await $`echo ${is_odd_patch2} > patches/is-odd@0.1.2.patch; ${bunExe()} i`.env(bunEnv).cwd(filedir); + + const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toContain("Hi from isOdd!\n"); + expect(stdout.toString()).not.toContain("lmao\n"); + }); + } + }); + it("should update a transitive dependency when the patchfile changes", async () => { $.throws(true); const filedir = tempDirWithFiles("patch1", { @@ -379,4 +431,56 @@ index c8950c17b265104bcf27f8c345df1a1b13a78950..7ce57ab96400ab0ff4fac7e06f6e02c2 expect(stderr.toString()).toBe(""); expect(stdout.toString()).toContain("lmao\n"); }); + + it("should update a scoped package", async () => { + const patchfile = /* patch */ `diff --git a/private/var/folders/wy/3969rv2x63g63jf8jwlcb2x40000gn/T/.b7f7d77b9ffdd3ee-00000000.tmp/index.js b/index.js +new file mode 100644 +index 0000000000000000000000000000000000000000..6edc0598a84632c41d9c770cfbbad7d99e2ab624 +--- /dev/null ++++ b/index.js +@@ -0,0 +1,4 @@ ++ ++module.exports = () => { ++ return 'PATCHED!' ++} +diff --git a/package.json b/package.json +index aa7c7012cda790676032d1b01d78c0b69ec06360..6048e7cb462b3f9f6ac4dc21aacf9a09397cd4be 100644 +--- a/package.json ++++ b/package.json +@@ -2,7 +2,7 @@ + "name": "@zackradisic/hls-dl", + "version": "0.0.1", + "description": "", +- "main": "dist/hls-dl.commonjs2.js", ++ "main": "./index.js", + "dependencies": { + "m3u8-parser": "^4.5.0", + "typescript": "^4.0.5" +`; + + $.throws(true); + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "patchedDependencies": { + "@zackradisic/hls-dl@0.0.1": "patches/thepatch.patch", + }, + "dependencies": { + "@zackradisic/hls-dl": "0.0.1", + }, + }), + patches: { + ["thepatch.patch"]: patchfile, + }, + "index.ts": /* ts */ `import hlsDl from '@zackradisic/hls-dl'; console.log(hlsDl())`, + }); + + await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + + const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toContain("PATCHED!\n"); + }); }); From e4b0376b7e73b58c719892b091b5f235529f2299 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Sun, 9 Jun 2024 15:16:30 -0700 Subject: [PATCH 02/51] rename concurrency resilient --- src/install/extract_tarball.zig | 57 ++++++--------------------------- src/install/install.zig | 10 +++--- src/install/patch_install.zig | 54 ++++--------------------------- src/sys.zig | 45 ++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 100 deletions(-) diff --git a/src/install/extract_tarball.zig b/src/install/extract_tarball.zig index 2635fe9e8d4159..0f856cdfb9d36f 100644 --- a/src/install/extract_tarball.zig +++ b/src/install/extract_tarball.zig @@ -285,7 +285,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD else => unreachable, }; if (folder_name.len == 0 or (folder_name.len == 1 and folder_name[0] == '/')) @panic("Tried to delete root and stopped it"); - var cache_dir = this.cache_dir; + const cache_dir = this.cache_dir; // e.g. @next // if it's a namespace package, we need to make sure the @name folder exists @@ -394,52 +394,15 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD } } - var did_atomically_replace = false; - if (did_atomically_replace and PackageManager.using_fallback_temp_dir) tmpdir.deleteTree(src) catch {}; - - attempt_atomic_rename_and_fallback_to_racy_delete: { - { - // Happy path: the folder doesn't exist in the cache dir, so we can - // just rename it. We don't need to delete anything. - var err = switch (bun.sys.renameat2(bun.toFD(tmpdir.fd), src, bun.toFD(cache_dir.fd), folder_name, .{ - .exclude = true, - })) { - .err => |err| err, - .result => break :attempt_atomic_rename_and_fallback_to_racy_delete, - }; - - // Fallback path: the folder exists in the cache dir, it might be in a strange state - // let's attempt to atomically replace it with the temporary folder's version - if (switch (err.getErrno()) { - .EXIST, .NOTEMPTY, .OPNOTSUPP => true, - else => false, - }) { - did_atomically_replace = true; - switch (bun.sys.renameat2(bun.toFD(tmpdir.fd), src, bun.toFD(cache_dir.fd), folder_name, .{ - .exchange = true, - })) { - .err => {}, - .result => break :attempt_atomic_rename_and_fallback_to_racy_delete, - } - did_atomically_replace = false; - } - } - - // sad path: let's try to delete the folder and then rename it - cache_dir.deleteTree(src) catch {}; - switch (bun.sys.renameat(bun.toFD(tmpdir.fd), src, bun.toFD(cache_dir.fd), folder_name)) { - .err => |err| { - this.package_manager.log.addErrorFmt( - null, - logger.Loc.Empty, - this.package_manager.allocator, - "moving \"{s}\" to cache dir failed: {}\n From: {s}\n To: {s}", - .{ name, err, tmpname, folder_name }, - ) catch unreachable; - return error.InstallFailed; - }, - .result => {}, - } + if (bun.sys.renameatConcurrently(bun.toFD(tmpdir.fd), src, bun.toFD(cache_dir.fd), folder_name).asErr()) |err| { + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "moving \"{s}\" to cache dir failed: {}\n From: {s}\n To: {s}", + .{ name, err, tmpname, folder_name }, + ) catch unreachable; + return error.InstallFailed; } } diff --git a/src/install/install.zig b/src/install/install.zig index 4004f028939103..e83843355456af 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -2309,9 +2309,7 @@ pub fn NewPackageInstall(comptime kind: PkgInstallKind) type { }; } - fn patchedPackageMissingFromCache(this: *@This(), manager: *PackageManager, package_id: PackageID, patchfile_hash: u64) bool { - _ = patchfile_hash; // autofix - + fn patchedPackageMissingFromCache(this: *@This(), manager: *PackageManager, package_id: PackageID) bool { // const patch_hash_prefix = "_patch_hash="; // var patch_hash_part_buf: [patch_hash_prefix.len + max_buntag_hash_buf_len + 1]u8 = undefined; // @memcpy(patch_hash_part_buf[0..patch_hash_prefix.len], patch_hash_prefix); @@ -10424,12 +10422,12 @@ pub const PackageManager = struct { } // rename to patches dir - if (bun.sys.renameat2( + + if (bun.sys.renameatConcurrently( bun.toFD(tmpdir.fd), tempfile_name, bun.FD.cwd(), path_in_patches_dir, - .{ .exclude = true }, ).asErr()) |e| { Output.prettyError( "error: failed to renaming patch file to patches dir {}\n", @@ -11060,7 +11058,7 @@ pub const PackageManager = struct { } const patch_patch, const patch_contents_hash, const patch_name_and_version_hash, const remove_patch = brk: { - if (this.manager.lockfile.patched_dependencies.entries.len == 0 and this.manager.patched_dependencies_to_remove.entries.len) break :brk .{ null, null, null, false }; + if (this.manager.lockfile.patched_dependencies.entries.len == 0 and this.manager.patched_dependencies_to_remove.entries.len == 0) break :brk .{ null, null, null, false }; var sfb = std.heap.stackFallback(1024, this.lockfile.allocator); const name_and_version = std.fmt.allocPrint(sfb.get(), "{s}@{s}", .{ name, package_version }) catch unreachable; defer sfb.get().free(name_and_version); diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index bbdd7e8e0fead9..cf50e287199f5c 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -399,53 +399,13 @@ pub const PatchTask = struct { }, .auto, ); - // var allocated = false; - // const package_name_z = brk: { - // if (this.package_name.len < tmpname_buf.len) { - // @memcpy(tmpname_buf[0..this.package_name.len], this.package_name); - // tmpname_buf[this.package_name.len] = 0; - // break :brk tmpname_buf[0..this.package_name.len :0]; - // } - // allocated = true; - // break :brk this.manager.allocator.dupeZ(u8, this.package_name) catch bun.outOfMemory(); - // }; - // defer if (allocated) this.manager.allocator.free(package_name_z); - - worked: { - if (bun.sys.renameat2( - bun.toFD(system_tmpdir.fd), - path_in_tmpdir, - bun.toFD(this.callback.apply.cache_dir.fd), - this.callback.apply.cache_dir_subpath, - .{ - .exclude = true, - }, - ).asErr()) |e_| { - var e = e_; - - if (if (comptime bun.Environment.isWindows) switch (e.getErrno()) { - bun.C.E.NOTEMPTY, bun.C.E.EXIST => true, - else => false, - } else switch (e.getErrno()) { - bun.C.E.NOTEMPTY, bun.C.E.EXIST, bun.C.E.OPNOTSUPP => true, - else => false, - }) { - switch (bun.sys.renameat2( - bun.toFD(system_tmpdir.fd), - path_in_tmpdir, - bun.toFD(this.callback.apply.cache_dir.fd), - this.callback.apply.cache_dir_subpath, - .{ - .exchange = true, - }, - )) { - .err => |ee| e = ee, - .result => break :worked, - } - } - return try log.addErrorFmtNoLoc(this.manager.allocator, "{}", .{e}); - } - } + + if (bun.sys.renameatConcurrently( + bun.toFD(system_tmpdir.fd), + path_in_tmpdir, + bun.toFD(this.callback.apply.cache_dir.fd), + this.callback.apply.cache_dir_subpath, + ).asErr()) |e| return try log.addErrorFmtNoLoc(this.manager.allocator, "{}", .{e}); } pub fn calcHash(this: *PatchTask) Maybe(u64) { diff --git a/src/sys.zig b/src/sys.zig index d9a4ce3bc40942..53b8483727b280 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -1683,6 +1683,51 @@ pub const RenameAt2Flags = packed struct { } }; +pub fn renameatConcurrently(from_dir_fd: bun.FileDescriptor, from: [:0]const u8, to_dir_fd: bun.FileDescriptor, to: [:0]const u8) Maybe(void) { + var did_atomically_replace = false; + + attempt_atomic_rename_and_fallback_to_racy_delete: { + { + // Happy path: the folder doesn't exist in the cache dir, so we can + // just rename it. We don't need to delete anything. + var err = switch (bun.sys.renameat2(from_dir_fd, from, to_dir_fd, to, .{ + .exclude = true, + })) { + .err => |err| err, + .result => break :attempt_atomic_rename_and_fallback_to_racy_delete, + }; + + // Fallback path: the folder exists in the cache dir, it might be in a strange state + // let's attempt to atomically replace it with the temporary folder's version + if (switch (err.getErrno()) { + .EXIST, .NOTEMPTY, .OPNOTSUPP => true, + else => false, + }) { + did_atomically_replace = true; + switch (bun.sys.renameat2(from_dir_fd, from, to_dir_fd, to, .{ + .exchange = true, + })) { + .err => {}, + .result => break :attempt_atomic_rename_and_fallback_to_racy_delete, + } + did_atomically_replace = false; + } + } + + // sad path: let's try to delete the folder and then rename it + var to_dir = to_dir_fd.asDir(); + to_dir.deleteTree(from) catch {}; + switch (bun.sys.renameat(from_dir_fd, from, to_dir_fd, to)) { + .err => |err| { + return .{ .err = err }; + }, + .result => {}, + } + } + + return Maybe(void).success; +} + pub fn renameat2(from_dir: bun.FileDescriptor, from: [:0]const u8, to_dir: bun.FileDescriptor, to: [:0]const u8, flags: RenameAt2Flags) Maybe(void) { if (Environment.isWindows) { return renameat(from_dir, from, to_dir, to); From 26631abfedd9eed8d14258333fbd6ee3f8c6c8fe Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:05:12 -0700 Subject: [PATCH 03/51] Fix a bunch of stuff --- src/install/install.zig | 559 ++++++++++++++++++++++++-------- src/js/internal-for-testing.ts | 1 + src/patch.zig | 67 +++- test/js/bun/patch/patch.test.ts | 45 ++- 4 files changed, 538 insertions(+), 134 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index e83843355456af..de0938ed46fbcd 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -9662,7 +9662,7 @@ pub const PackageManager = struct { } } - const updates = UpdateRequest.parse(ctx.allocator, ctx.log, manager.options.positionals[1..], &update_requests, manager.subcommand); + const updates: []UpdateRequest = if (manager.subcommand == .@"patch-commit") &[_]UpdateRequest{} else UpdateRequest.parse(ctx.allocator, ctx.log, manager.options.positionals[1..], &update_requests, manager.subcommand); switch (manager.subcommand) { inline else => |subcommand| try manager.updatePackageJSONAndInstallWithManagerWithUpdates( ctx, @@ -9800,21 +9800,22 @@ pub const PackageManager = struct { } }, .@"patch-commit" => { - _ = manager.lockfile.loadFromDisk( - manager, - manager.allocator, - manager.log, - manager.options.lockfile_path, - true, - ); + // _ = manager.lockfile.loadFromDisk( + // manager, + // manager.allocator, + // manager.log, + // manager.options.lockfile_path, + // true, + // ); var pathbuf: bun.PathBuffer = undefined; - const stuff = try manager.doPatchCommit(&pathbuf, log_level); - try PackageJSONEditor.editPatchedDependencies( - manager, - ¤t_package_json.root, - stuff.patch_key, - stuff.patchfile_path, - ); + if (try manager.doPatchCommit(&pathbuf, log_level)) |stuff| { + try PackageJSONEditor.editPatchedDependencies( + manager, + ¤t_package_json.root, + stuff.patch_key, + stuff.patchfile_path, + ); + } }, .link, .add, .update => { // `bun update ` is basically the same as `bun add `, except @@ -10027,166 +10028,466 @@ pub const PackageManager = struct { } } + fn nodeModulesFolderForDependencyID(iterator: *Lockfile.Tree.Iterator, dependency_id: DependencyID) !?Lockfile.Tree.NodeModulesFolder { + while (iterator.nextNodeModulesFolder(null)) |node_modules| { + _ = std.mem.indexOfScalar(DependencyID, node_modules.dependencies, dependency_id) orelse continue; + return node_modules; + } + + return null; + } + + fn pkg_dep_id_for_name_and_version( + lockfile: *Lockfile, + pkg_maybe_version_to_patch: []const u8, + name: []const u8, + version: ?[]const u8, + ) struct { PackageID, DependencyID } { + const name_hash = String.Builder.stringHash(name); + + const strbuf = lockfile.buffers.string_bytes.items; + + const dependency_id: DependencyID, const pkg_id: PackageID = brk: { + var buf: [1024]u8 = undefined; + const dependencies = lockfile.buffers.dependencies.items; + + var matches_found: u32 = 0; + var maybe_first_match: ?struct { DependencyID, PackageID } = null; + for (dependencies, 0..) |dep, dep_id| { + if (dep.name_hash != name_hash) continue; + matches_found += 1; + const pkg_id = lockfile.buffers.resolutions.items[dep_id]; + const pkg = lockfile.packages.get(pkg_id); + if (version) |v| { + const label = std.fmt.bufPrint(buf[0..], "{}", .{pkg.resolution.fmt(strbuf, .posix)}) catch @panic("Resolution name too long"); + if (std.mem.eql(u8, label, v)) break :brk .{ @intCast(dep_id), pkg_id }; + } else maybe_first_match = .{ @intCast(dep_id), pkg_id }; + } + + const first_match = maybe_first_match orelse { + Output.prettyErrorln("\nerror: package {s} not found", .{pkg_maybe_version_to_patch}); + Output.flush(); + Global.crash(); + return; + }; + + if (matches_found > 1) { + Output.prettyErrorln( + "\nerror: please specify a precise version:", + .{}, + ); + var i: usize = 0; + const pkg_hashes = lockfile.packages.items(.name_hash); + while (i < lockfile.packages.len) { + if (std.mem.indexOfScalar(u64, pkg_hashes[i..], name_hash)) |idx| { + defer i += idx + 1; + const pkg_id = i + idx; + const pkg = lockfile.packages.get(pkg_id); + if (!std.mem.eql(u8, pkg.name.slice(strbuf), name)) continue; + + Output.prettyError(" {s}@{}\n", .{ pkg.name.slice(strbuf), pkg.resolution.fmt(strbuf, .posix) }); + } else break; + } + Output.flush(); + Global.crash(); + return; + } + + break :brk .{ first_match[0], first_match[1] }; + }; + + return .{ pkg_id, dependency_id }; + } + /// - Arg is name and possibly version (e.g. "is-even" or "is-even@1.0.0") /// - Find package that satisfies name and version /// - Copy contents of package into temp dir /// - Give that to user fn preparePatch(manager: *PackageManager) !void { - const @"pkg + maybe version to patch" = manager.options.positionals[1]; + const strbuf = manager.lockfile.buffers.string_bytes.items; + _ = strbuf; // autofix + const pkg_maybe_version_to_patch = manager.options.positionals[1]; + const name: []const u8, const version: ?[]const u8 = brk: { - if (std.mem.indexOfScalar(u8, @"pkg + maybe version to patch"[1..], '@')) |version_delimiter| { + if (std.mem.indexOfScalar(u8, pkg_maybe_version_to_patch[1..], '@')) |version_delimiter| { break :brk .{ - @"pkg + maybe version to patch"[0..version_delimiter], - @"pkg + maybe version to patch"[1..][version_delimiter + 1 ..], + pkg_maybe_version_to_patch[0..version_delimiter], + pkg_maybe_version_to_patch[1..][version_delimiter + 1 ..], }; } break :brk .{ - @"pkg + maybe version to patch", + pkg_maybe_version_to_patch, null, }; }; - const name_hash = String.Builder.stringHash(name); + const result = pkg_dep_id_for_name_and_version(manager.lockfile, pkg_maybe_version_to_patch, name, version); + const pkg_id = result[0]; + _ = pkg_id; // autofix + const dependency_id = result[1]; - const strbuf = manager.lockfile.buffers.string_bytes.items; + var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); + const folder = (try nodeModulesFolderForDependencyID(&iterator, dependency_id)) orelse { + @panic("TODO zack: bun patch node_modules folder not found"); + }; - const pkg_id: u64 = brk: { - var buf: [1024]u8 = undefined; - var i: usize = 0; - - const pkg_hashes = manager.lockfile.packages.items(.name_hash); - var matches_count: u32 = 0; - var first_match: ?u64 = null; - while (i < manager.lockfile.packages.len) { - if (std.mem.indexOfScalar(u64, pkg_hashes[i..], name_hash)) |idx| { - defer i += idx + 1; - const pkg_id = i + idx; - const pkg = manager.lockfile.packages.get(pkg_id); - const pkg_name = pkg.name.slice(strbuf); - if (!std.mem.eql(u8, pkg_name, name)) continue; - matches_count += 1; - - // if they supplied a version it needs to match it, - // otherwise we'll just pick the first one we see, if there are multiple we throw error - if (version) |v| { - const label = std.fmt.bufPrint(buf[0..], "{}", .{pkg.resolution.fmt(strbuf, .posix)}) catch @panic("Resolution name too long"); - if (std.mem.eql(u8, label, v)) break :brk pkg_id; - } else { - first_match = pkg_id; + const module_folder = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); + Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ name, module_folder }); + Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch-commit '{s}'\n", .{module_folder}); + + return; + } + + const PatchCommitResult = struct { + patch_key: []const u8, + patchfile_path: []const u8, + }; + + /// - Arg is the dir containing the package with changes OR name and version + /// - Get the patch file contents by running git diff on the temp dir and the original package dir + /// - Write the patch file to $PATCHES_DIR/$PKG_NAME_AND_VERSION.patch + /// - Update "patchedDependencies" in package.json + /// - Run install to install newly patched pkg + fn doPatchCommit( + manager: *PackageManager, + pathbuf: *bun.PathBuffer, + comptime log_level: Options.LogLevel, + ) !?PatchCommitResult { + var lockfile: *Lockfile = try manager.allocator.create(Lockfile); + defer lockfile.deinit(); + switch (lockfile.loadFromDisk(manager, manager.allocator, manager.log, manager.options.lockfile_path, true)) { + .not_found => { + Output.panic("Lockfile not found", .{}); + }, + .err => |cause| { + if (log_level != .silent) { + switch (cause.step) { + .open_file => Output.prettyError("error opening lockfile: {s}\n", .{ + @errorName(cause.value), + }), + .parse_file => Output.prettyError("error parsing lockfile: {s}\n", .{ + @errorName(cause.value), + }), + .read_file => Output.prettyError("error reading lockfile: {s}\n", .{ + @errorName(cause.value), + }), + .migrating => Output.prettyError("error migrating lockfile: {s}\n", .{ + @errorName(cause.value), + }), } - } else break; - } - if (first_match) |id| { - if (matches_count > 1) { - Output.prettyErrorln( - "\nerror: please specify a precise version:", - .{}, - ); - i = 0; - while (i < manager.lockfile.packages.len) { - if (std.mem.indexOfScalar(u64, pkg_hashes[i..], name_hash)) |idx| { - defer i += idx + 1; - const pkg_id = i + idx; - const pkg = manager.lockfile.packages.get(pkg_id); - if (!std.mem.eql(u8, pkg.name.slice(strbuf), name)) continue; - Output.prettyError(" {s}@{}\n", .{ pkg.name.slice(strbuf), pkg.resolution.fmt(strbuf, .posix) }); - } else break; + if (manager.options.enable.fail_early) { + Output.prettyError("failed to load lockfile\n", .{}); + } else { + Output.prettyError("ignoring lockfile\n", .{}); } + Output.flush(); - Global.crash(); - return; } - break :brk id; - } - Output.prettyErrorln( - "\nerror: could not find package: {s}\n", - .{@"pkg + maybe version to patch"}, - ); - Output.flush(); - return; - }; + Global.crash(); + }, + .ok => {}, + } - const pkg = manager.lockfile.packages.get(pkg_id); + const ArgKind = enum { + path, + name_and_version, + }; - const resolution: *const Resolution = &manager.lockfile.packages.items(.resolution)[pkg_id]; - const stuff = manager.computeCacheDirAndSubpath(name, resolution, null); - const cache_dir_subpath: [:0]const u8 = stuff.cache_dir_subpath; - const cache_dir: std.fs.Dir = stuff.cache_dir; + const argument = manager.options.positionals[1]; - // copy the contents into a tempdir - var tmpname_buf: [1024]u8 = undefined; - const tempdir_name = bun.span(try bun.fs.FileSystem.instance.tmpname("tmp", &tmpname_buf, bun.fastRandom())); - const tmpdir = try bun.fs.FileSystem.instance.tmpdir(); - var destination_dir = try tmpdir.makeOpenPath(tempdir_name, .{}); - defer destination_dir.close(); - - var resolution_buf: [512]u8 = undefined; - const resolution_label = std.fmt.bufPrint(&resolution_buf, "{}", .{pkg.resolution.fmt(strbuf, .posix)}) catch unreachable; - const dummy_node_modules = .{ - .path = std.ArrayList(u8).init(manager.allocator), - .tree_id = 0, + const arg_kind: ArgKind = brk: { + if (bun.strings.hasPrefix(argument, "node_modules/")) break :brk .path; + if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules/")) break :brk .path; + break :brk .name_and_version; }; - var pkg_install = PreparePatchPackageInstall{ - .allocator = manager.allocator, - .cache_dir = cache_dir, - .cache_dir_subpath = cache_dir_subpath, - .destination_dir_subpath = tempdir_name, - .destination_dir_subpath_buf = tmpname_buf[0..], - .progress = .{}, - .package_name = name, - .package_version = resolution_label, - // dummy value - .node_modules = &dummy_node_modules, + + var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); + var resolution_buf: [1024]u8 = undefined; + const _cache_dir: std.fs.Dir, const _cache_dir_subpath: stringZ, const _changes_dir: []const u8, const _pkg: Package = switch (arg_kind) { + .path => result: { + const package_json_source: logger.Source = brk: { + const package_json_path = bun.path.joinZ(&[_][]const u8{ argument, "package.json" }, .auto); + + switch (bun.sys.File.toSource(package_json_path, manager.allocator)) { + .result => |s| break :brk s, + .err => |e| { + Output.prettyError( + "error: failed to read package.json: {}\n", + .{e.withPath(package_json_path).toSystemError()}, + ); + Output.flush(); + Global.crash(); + }, + } + }; + defer manager.allocator.free(package_json_source.contents); + + initializeStore(); + const json = json_parser.ParsePackageJSONUTF8AlwaysDecode(&package_json_source, manager.log, manager.allocator) catch |err| { + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; + }, + } + Output.prettyErrorln("{s} parsing package.json in \"{s}\"", .{ @errorName(err), package_json_source.path.prettyDir() }); + Global.crash(); + }; + + const version = version: { + if (json.asProperty("version")) |v| { + if (v.expr.asString(manager.allocator)) |s| break :version s; + } + Output.prettyError( + "error: invalid package.json, missing or invalid property \"version\": {s}\n", + .{package_json_source.path.text}, + ); + Output.flush(); + Global.crash(); + }; + + var package = Lockfile.Package{}; + try package.parseWithJSON(lockfile, manager.allocator, manager.log, package_json_source, json, void, {}, Features.folder); + + const name = lockfile.str(&package.name); + const actual_package = switch (lockfile.package_index.get(package.name_hash) orelse { + Output.prettyError( + "error: failed to find package in lockfile package index, this is a bug in Bun. Please file a GitHub issue.\n", + .{}, + ); + Output.flush(); + Global.crash(); + }) { + .PackageID => |id| lockfile.packages.get(id), + .PackageIDMultiple => |ids| brk: { + for (ids.items) |id| { + const pkg = lockfile.packages.get(id); + const resolution_label = std.fmt.bufPrint(&resolution_buf, "{}", .{pkg.resolution.fmt(lockfile.buffers.string_bytes.items, .posix)}) catch unreachable; + if (std.mem.eql(u8, resolution_label, version)) { + break :brk pkg; + } + } + Output.prettyError("error: could not find package with name: {s}\n", .{ + package.name.slice(lockfile.buffers.string_bytes.items), + }); + Output.flush(); + Global.crash(); + }, + }; + + const cache_result = manager.computeCacheDirAndSubpath(name, &actual_package.resolution, null); + const cache_dir = cache_result.cache_dir; + const cache_dir_subpath = cache_result.cache_dir_subpath; + + const changes_dir = argument; + + break :result .{ cache_dir, cache_dir_subpath, changes_dir, actual_package }; + }, + .name_and_version => brk: { + const name: []const u8, const version: ?[]const u8 = brk1: { + if (std.mem.indexOfScalar(u8, argument[1..], '@')) |version_delimiter| { + break :brk1 .{ + argument[0..version_delimiter], + argument[1..][version_delimiter + 1 ..], + }; + } + break :brk1 .{ + argument, + null, + }; + }; + const result = pkg_dep_id_for_name_and_version(lockfile, argument, name, version); + const pkg_id: PackageID = result[0]; + const dependency_id: DependencyID = result[1]; + const node_modules = (try nodeModulesFolderForDependencyID( + &iterator, + dependency_id, + )) orelse { + @panic("TODO zack node_modules not found"); + }; + const changes_dir = bun.path.joinZBuf(pathbuf[0..], &[_][]const u8{ + node_modules.relative_path, + name, + }, .auto); + const pkg = lockfile.packages.get(pkg_id); + const cache_result = manager.computeCacheDirAndSubpath(pkg.name.slice(lockfile.buffers.string_bytes.items), &pkg.resolution, null); + const cache_dir = cache_result.cache_dir; + const cache_dir_subpath = cache_result.cache_dir_subpath; + break :brk .{ cache_dir, cache_dir_subpath, changes_dir, pkg }; + }, }; - switch (pkg_install.installWithMethod(true, tmpdir, .copyfile)) { - .success => {}, - .fail => |reason| { - Output.prettyErrorln( - "\nerror: failed to copy package to temp directory: {s}, during step: {s}\n", - .{ - @errorName(reason.err), - reason.step.name(), + // zls + const cache_dir: std.fs.Dir = _cache_dir; + const cache_dir_subpath: stringZ = _cache_dir_subpath; + const changes_dir: []const u8 = _changes_dir; + const pkg: Package = _pkg; + + const name = pkg.name.slice(lockfile.buffers.string_bytes.items); + const resolution_label = std.fmt.bufPrint(&resolution_buf, "{s}@{}", .{ name, pkg.resolution.fmt(lockfile.buffers.string_bytes.items, .posix) }) catch unreachable; + + const patchfile_contents = brk: { + const new_folder = changes_dir; + var buf2: bun.PathBuffer = undefined; + const old_folder = old_folder: { + const cache_dir_path = switch (bun.sys.getFdPath(bun.toFD(cache_dir.fd), &buf2)) { + .result => |s| s, + .err => |e| { + Output.prettyError( + "error: failed to read from cache {}\n", + .{e.toSystemError()}, + ); + Output.flush(); + Global.crash(); }, + }; + break :old_folder bun.path.join(&[_][]const u8{ + cache_dir_path, + cache_dir_subpath, + }, .posix); + }; + std.debug.print("OLD FOLDER: {s}\n", .{old_folder}); + std.debug.print("NEW FOLDER: {s}\n", .{new_folder}); + const contents = switch (bun.patch.gitDiff(manager.allocator, old_folder, new_folder) catch |e| { + Output.prettyError( + "error: failed to make diff {s}\n", + .{@errorName(e)}, ); Output.flush(); - return; - }, - } + Global.crash(); + }) { + .result => |stdout| stdout, + .err => |stderr| { + defer stderr.deinit(); + const Truncate = struct { + stderr: std.ArrayList(u8), - var pathbuf: bun.PathBuffer = undefined; - const pkg_to_patch_dir = switch (bun.sys.getFdPath(bun.toFD(destination_dir.fd), &pathbuf)) { + pub fn format( + this: *const @This(), + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) !void { + const truncate_stderr = this.stderr.items.len > 256; + if (truncate_stderr) { + try writer.print("{s}... ({d} more bytes)", .{ this.stderr.items[0..256], this.stderr.items.len - 256 }); + } else try writer.print("{s}", .{this.stderr.items[0..]}); + } + }; + Output.prettyError( + "error: failed to make diff {}\n", + .{ + Truncate{ .stderr = stderr }, + }, + ); + Output.flush(); + Global.crash(); + }, + }; + + if (contents.items.len == 0) { + Output.pretty("\nNo changes detected, comparing {s} to {s}\n", .{ old_folder, new_folder }); + Output.flush(); + contents.deinit(); + return null; + } + + break :brk contents; + }; + defer patchfile_contents.deinit(); + + // write the patch contents to temp file then rename + var tmpname_buf: [1024]u8 = undefined; + const tempfile_name = bun.span(try bun.fs.FileSystem.instance.tmpname("tmp", &tmpname_buf, bun.fastRandom())); + const tmpdir = try bun.fs.FileSystem.instance.tmpdir(); + const tmpfd = switch (bun.sys.openat( + bun.toFD(tmpdir.fd), + tempfile_name, + std.os.O.RDWR | std.os.O.CREAT, + 0o666, + )) { .result => |fd| fd, .err => |e| { - Output.prettyErrorln( - "\nerror: {}\n", - .{ - e.toSystemError(), - }, + Output.prettyError( + "error: failed to open temp file {}\n", + .{e.toSystemError()}, ); Output.flush(); - return; + Global.crash(); }, }; + defer _ = bun.sys.close(tmpfd); - Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ name, pkg_to_patch_dir }); - Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch-commit '{s}'\n", .{pkg_to_patch_dir}); + if (bun.sys.File.writeAll(.{ .handle = tmpfd }, patchfile_contents.items).asErr()) |e| { + Output.prettyError( + "error: failed to write patch to temp file {}\n", + .{e.toSystemError()}, + ); + Output.flush(); + Global.crash(); + } - return; - } + @memcpy(resolution_buf[resolution_label.len .. resolution_label.len + ".patch".len], ".patch"); + var patch_filename: []const u8 = resolution_buf[0 .. resolution_label.len + ".patch".len]; + var deinit = false; + if (escapePatchFilename(manager.allocator, patch_filename)) |escaped| { + deinit = true; + patch_filename = escaped; + } + defer if (deinit) manager.allocator.free(patch_filename); - const PatchCommitResult = struct { - patch_key: []const u8, - patchfile_path: []const u8, - }; + const path_in_patches_dir = bun.path.joinZ( + &[_][]const u8{ + manager.options.patch_features.commit.patches_dir, + patch_filename, + }, + .posix, + ); + + var nodefs = bun.JSC.Node.NodeFS{}; + const args = bun.JSC.Node.Arguments.Mkdir{ + .path = .{ .string = bun.PathString.init(manager.options.patch_features.commit.patches_dir) }, + }; + if (nodefs.mkdirRecursive(args, .sync).asErr()) |e| { + Output.prettyError( + "error: failed to make patches dir {}\n", + .{e.toSystemError()}, + ); + Output.flush(); + Global.crash(); + } + + // rename to patches dir + if (bun.sys.renameatConcurrently( + bun.toFD(tmpdir.fd), + tempfile_name, + bun.FD.cwd(), + path_in_patches_dir, + ).asErr()) |e| { + Output.prettyError( + "error: failed to renaming patch file to patches dir {}\n", + .{e.toSystemError()}, + ); + Output.flush(); + Global.crash(); + } + + const patch_key = std.fmt.allocPrint(manager.allocator, "{s}", .{resolution_label}) catch bun.outOfMemory(); + const patchfile_path = manager.allocator.dupe(u8, path_in_patches_dir) catch bun.outOfMemory(); + _ = bun.sys.unlink(bun.path.joinZ(&[_][]const u8{ changes_dir, ".bun-patch-tag" }, .auto)); + + return .{ + .patch_key = patch_key, + .patchfile_path = patchfile_path, + }; + } /// - Arg is the tempdir containing the package with changes /// - Get the patch file contents by running git diff on the temp dir and the original package dir /// - Write the patch file to $PATCHES_DIR/$PKG_NAME_AND_VERSION.patch /// - Update "patchedDependencies" in package.json /// - Run install to install newly patched pkg - fn doPatchCommit( + fn doPatchCommitOld( manager: *PackageManager, pathbuf: *bun.PathBuffer, comptime log_level: Options.LogLevel, diff --git a/src/js/internal-for-testing.ts b/src/js/internal-for-testing.ts index 89e6c8758bbb4e..4c4d69eccafdfc 100644 --- a/src/js/internal-for-testing.ts +++ b/src/js/internal-for-testing.ts @@ -20,6 +20,7 @@ export const SQL = $cpp("JSSQLStatement.cpp", "createJSSQLStatementConstructor") export const patchInternals = { parse: $newZigFunction("patch.zig", "TestingAPIs.parse", 1), apply: $newZigFunction("patch.zig", "TestingAPIs.apply", 2), + makeDiff: $newZigFunction("patch.zig", "TestingAPIs.makeDiff", 2), }; export const shellInternals = { diff --git a/src/patch.zig b/src/patch.zig index 3c7e0f08f67bf1..eb10cd7b333fd0 100644 --- a/src/patch.zig +++ b/src/patch.zig @@ -9,6 +9,8 @@ const WHITESPACE: []const u8 = " \t\n\r"; // TODO: calculate this for different systems const PAGE_SIZE = 16384; +const debug = bun.Output.scoped(.patch, false); + /// All strings point to the original patch file text pub const PatchFilePart = union(enum) { file_patch: *FilePatch, @@ -534,7 +536,6 @@ pub const PatchFilePartKind = enum { }; const ParseErr = error{ - empty_patchfile, unrecognized_pragma, no_newline_at_eof_pragma_encountered_without_context, hunk_lines_encountered_before_hunk_header, @@ -755,7 +756,7 @@ const PatchLinesParser = struct { file_: []const u8, opts: struct { support_legacy_diffs: bool = false }, ) ParseErr!void { - if (file_.len == 0) return ParseErr.empty_patchfile; + if (file_.len == 0) return; const end = brk: { var iter = std.mem.splitBackwardsScalar(u8, file_, '\n'); var prev: usize = file_.len; @@ -1088,6 +1089,45 @@ const PatchLinesParser = struct { }; pub const TestingAPIs = struct { + pub fn makeDiff(globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) callconv(.C) JSC.JSValue { + const arguments_ = callframe.arguments(2); + var arguments = JSC.Node.ArgumentsSlice.init(globalThis.bunVM(), arguments_.slice()); + + const old_folder_jsval = arguments.nextEat() orelse { + globalThis.throw("expected 2 strings", .{}); + return .undefined; + }; + const old_folder_bunstr = old_folder_jsval.toBunString(globalThis); + defer old_folder_bunstr.deref(); + + const new_folder_jsval = arguments.nextEat() orelse { + globalThis.throw("expected 2 strings", .{}); + return .undefined; + }; + const new_folder_bunstr = new_folder_jsval.toBunString(globalThis); + defer new_folder_bunstr.deref(); + + const old_folder = old_folder_bunstr.toUTF8(bun.default_allocator); + defer old_folder.deinit(); + + const new_folder = new_folder_bunstr.toUTF8(bun.default_allocator); + defer new_folder.deinit(); + + return switch (gitDiff(bun.default_allocator, old_folder.slice(), new_folder.slice()) catch |e| { + globalThis.throwError(e, "failed to make diff"); + return .undefined; + }) { + .result => |s| { + defer s.deinit(); + return bun.String.fromBytes(s.items).toJS(globalThis); + }, + .err => |e| { + defer e.deinit(); + globalThis.throw("failed to make diff: {s}", .{e.items}); + return .undefined; + }, + }; + } const ApplyArgs = struct { patchfile_txt: JSC.ZigString.Slice, patchfile: PatchFile, @@ -1262,6 +1302,7 @@ pub fn gitDiff( return .{ .err = stderr }; } + debug("Before postprocess: {s}\n", .{stdout.items}); try gitDiffPostprocess(&stdout, old_folder, new_folder); deinit_stdout = false; return .{ .result = stdout }; @@ -1308,9 +1349,12 @@ fn gitDiffPostprocess(stdout: *std.ArrayList(u8), old_folder: []const u8, new_fo @memcpy(new_buf[2..][0..new_folder_trimmed.len], new_folder_trimmed); new_buf[2 + new_folder_trimmed.len] = '/'; - break :brk .{ old_buf[0 .. 2 + old_folder_trimmed.len + 1], new_buf[0 .. 2 + old_folder_trimmed.len + 1] }; + break :brk .{ old_buf[0 .. 2 + old_folder_trimmed.len + 1], new_buf[0 .. 2 + new_folder_trimmed.len + 1] }; }; + // const @"$old_folder/" = @"a/$old_folder/"[2..]; + // const @"$new_folder/" = @"b/$new_folder/"[2..]; + var line_iter = std.mem.splitScalar(u8, stdout.items, '\n'); while (line_iter.next()) |line| { if (shouldSkipLine(line)) continue; @@ -1326,6 +1370,23 @@ fn gitDiffPostprocess(stdout: *std.ArrayList(u8), old_folder: []const u8, new_fo const line_start = line_iter.index.? - 1 - line.len; try stdout.replaceRange(line_start + @"$new_folder/ start", new_folder_trimmed.len + 1, ""); line_iter.index.? -= new_folder_trimmed.len + 1; + continue; + } + if (std.mem.indexOf(u8, line, old_folder)) |idx| { + if (idx + old_folder.len < line.len and line[idx + old_folder.len] == '/') { + const line_start = line_iter.index.? - 1 - line.len; + line_iter.index.? -= 1 + line.len; + try stdout.replaceRange(line_start + idx, old_folder.len + 1, ""); + continue; + } + } + if (std.mem.indexOf(u8, line, new_folder)) |idx| { + if (idx + new_folder.len < line.len and line[idx + new_folder.len] == '/') { + const line_start = line_iter.index.? - 1 - line.len; + line_iter.index.? -= 1 + line.len; + try stdout.replaceRange(line_start + idx, new_folder.len + 1, ""); + continue; + } } } } diff --git a/test/js/bun/patch/patch.test.ts b/test/js/bun/patch/patch.test.ts index e3352f212ec734..e11c1f076e8d31 100644 --- a/test/js/bun/patch/patch.test.ts +++ b/test/js/bun/patch/patch.test.ts @@ -4,9 +4,9 @@ import { patchInternals } from "bun:internal-for-testing"; import { tempDirWithFiles as __tempDirWithFiles } from "harness"; import { join as __join } from "node:path"; import fs from "fs/promises"; -const { parse, apply } = patchInternals; +const { parse, apply, makeDiff } = patchInternals; -const makeDiff = async (aFolder: string, bFolder: string, cwd: string): Promise => { +const makeDiffJs = async (aFolder: string, bFolder: string, cwd: string): Promise => { const { stdout, stderr } = await $`git -c core.safecrlf=false diff --src-prefix=a/ --dst-prefix=b/ --ignore-cr-at-eol --irreversible-delete --full-index --no-index ${aFolder} ${bFolder}` .env( @@ -51,6 +51,47 @@ const join = : __join; describe("apply", () => { + test("edgecase", async () => { + const newcontents = "module.exports = x => x % 420 === 0;"; + const tempdir2 = tempDirWithFiles("patch-test2", { + ".bun/install/cache/is-even@1.0.0": { + "index.js": "module.exports = x => x % 2 === 0;", + }, + }); + const tempdir = tempDirWithFiles("patch-test", { + a: {}, + ["node_modules/is-even"]: { + "index.js": newcontents, + }, + }); + + const patchfile = await makeDiff( + `${tempdir2}/.bun/install/cache/is-even@1.0.0`, + `${tempdir}/node_modules/is-even`, + tempdir, + ); + + await apply(patchfile, `${tempdir}/node_modules/is-even`); + expect(await fs.readFile(`${tempdir}/node_modules/is-even/index.js`).then(b => b.toString())).toBe(newcontents); + }); + + test("empty", async () => { + const tempdir = tempDirWithFiles("patch-test", { + a: {}, + b: {}, + }); + + const afolder = join(tempdir, "a"); + const bfolder = join(tempdir, "b"); + + const patchfile = await makeDiff(afolder, bfolder, tempdir); + expect(patchfile).toBe(""); + + await apply(patchfile, afolder); + + expect(await fs.readdir(afolder)).toEqual([]); + }); + describe("deletion", () => { test("simple", async () => { const files = { From 1bf823aa378dda63b7fecb442b33315b4ff2f572 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:15:13 -0700 Subject: [PATCH 04/51] yoops --- test/cli/install/bun-install-patch.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/cli/install/bun-install-patch.test.ts b/test/cli/install/bun-install-patch.test.ts index 398e5929b6def4..6bf623a2668936 100644 --- a/test/cli/install/bun-install-patch.test.ts +++ b/test/cli/install/bun-install-patch.test.ts @@ -377,7 +377,6 @@ index c8950c17b265104bcf27f8c345df1a1b13a78950..7ce57ab96400ab0ff4fac7e06f6e02c2 console.log("FILEDIR", filedir); await $`${bunExe()} i`.env(bunEnv).cwd(filedir); - throw new Error("woopsie!"); await $`echo ${JSON.stringify({ "name": "bun-patch-test", @@ -393,7 +392,7 @@ index c8950c17b265104bcf27f8c345df1a1b13a78950..7ce57ab96400ab0ff4fac7e06f6e02c2 .env(bunEnv) .cwd(filedir); - await $`echo ${is_odd_patch2} > patches/is-odd@0.1.2.patch; ${bunExe()} i`.env(bunEnv).cwd(filedir); + await $`echo ${is_odd_patch} > patches/is-odd@0.1.2.patch; ${bunExe()} i`.env(bunEnv).cwd(filedir); const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(filedir); expect(stderr.toString()).toBe(""); From 639f0f32340838c10e2d2b7dba8c7e1519de1bad Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:52:44 -0700 Subject: [PATCH 05/51] remove some panics --- src/install/install.zig | 418 ++++++---------------------------- src/install/patch_install.zig | 2 + 2 files changed, 70 insertions(+), 350 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index de0938ed46fbcd..93caaf9295e263 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -3843,11 +3843,11 @@ pub const PackageManager = struct { manager: *PackageManager, pkg_name: string, resolution: *const Resolution, + folder_path_buf: *bun.PathBuffer, patch_hash: ?u64, ) struct { cache_dir: std.fs.Dir, cache_dir_subpath: stringZ } { const name = pkg_name; const buf = manager.lockfile.buffers.string_bytes.items; - _ = buf; // autofix var cache_dir = std.fs.cwd(); var cache_dir_subpath: stringZ = ""; @@ -3868,19 +3868,18 @@ pub const PackageManager = struct { cache_dir = manager.getCacheDirectory(); }, .folder => { - @panic("TODO @zack fix"); - // const folder = resolution.value.folder.slice(buf); - // // Handle when a package depends on itself via file: - // // example: - // // "mineflayer": "file:." - // if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { - // cache_dir_subpath = "."; - // } else { - // @memcpy(manager.folder_path_buf[0..folder.len], folder); - // this.folder_path_buf[folder.len] = 0; - // cache_dir_subpath = this.folder_path_buf[0..folder.len :0]; - // } - // cache_dir = std.fs.cwd(); + const folder = resolution.value.folder.slice(buf); + // Handle when a package depends on itself via file: + // example: + // "mineflayer": "file:." + if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { + cache_dir_subpath = "."; + } else { + @memcpy(folder_path_buf[0..folder.len], folder); + folder_path_buf[folder.len] = 0; + cache_dir_subpath = folder_path_buf[0..folder.len :0]; + } + cache_dir = std.fs.cwd(); }, .local_tarball => { cache_dir_subpath = manager.cachedTarballFolderName(resolution.value.local_tarball, patch_hash); @@ -3891,77 +3890,51 @@ pub const PackageManager = struct { cache_dir = manager.getCacheDirectory(); }, .workspace => { - @panic("TODO @zack fix"); - // const folder = resolution.value.workspace.slice(buf); - // // Handle when a package depends on itself - // if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { - // cache_dir_subpath = "."; - // } else { - // @memcpy(this.folder_path_buf[0..folder.len], folder); - // this.folder_path_buf[folder.len] = 0; - // cache_dir_subpath = this.folder_path_buf[0..folder.len :0]; - // } - // cache_dir = std.fs.cwd(); + const folder = resolution.value.workspace.slice(buf); + // Handle when a package depends on itself + if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { + cache_dir_subpath = "."; + } else { + @memcpy(folder_path_buf[0..folder.len], folder); + folder_path_buf[folder.len] = 0; + cache_dir_subpath = folder_path_buf[0..folder.len :0]; + } + cache_dir = std.fs.cwd(); }, .symlink => { - @panic("TODO @zack fix"); - // const directory = manager.globalLinkDir() catch |err| { - // if (comptime log_level != .silent) { - // const fmt = "\nerror: unable to access global directory while installing {s}: {s}\n"; - // const args = .{ name, @errorName(err) }; - - // if (comptime log_level.showProgress()) { - // switch (Output.enable_ansi_colors) { - // inline else => |enable_ansi_colors| { - // this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); - // }, - // } - // } else { - // Output.prettyErrorln(fmt, args); - // } - // } - - // if (manager.options.enable.fail_early) { - // Global.exit(1); - // } - - // Output.flush(); - // this.summary.fail += 1; - // this.incrementTreeInstallCount(this.current_tree_id, !is_pending_package_install, log_level); - // return; - // }; - - // const folder = resolution.value.symlink.slice(buf); - - // if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { - // cache_dir_subpath = "."; - // cache_dir = std.fs.cwd(); - // } else { - // const global_link_dir = manager.globalLinkDirPath() catch unreachable; - // var ptr = &this.folder_path_buf; - // var remain: []u8 = this.folder_path_buf[0..]; - // @memcpy(ptr[0..global_link_dir.len], global_link_dir); - // remain = remain[global_link_dir.len..]; - // if (global_link_dir[global_link_dir.len - 1] != std.fs.path.sep) { - // remain[0] = std.fs.path.sep; - // remain = remain[1..]; - // } - // @memcpy(remain[0..folder.len], folder); - // remain = remain[folder.len..]; - // remain[0] = 0; - // const len = @intFromPtr(remain.ptr) - @intFromPtr(ptr); - // cache_dir_subpath = this.folder_path_buf[0..len :0]; - // cache_dir = directory; - // } - }, - else => { - @panic("TODO @zack fix"); - // if (comptime Environment.allow_assert) { - // @panic("Internal assertion failure: unexpected resolution tag"); - // } - // this.incrementTreeInstallCount(this.current_tree_id, !is_pending_package_install, log_level); - // return; + const directory = manager.globalLinkDir() catch |err| { + const fmt = "\nerror: unable to access global directory while installing {s}: {s}\n"; + const args = .{ name, @errorName(err) }; + + Output.prettyErrorln(fmt, args); + + Global.exit(1); + }; + + const folder = resolution.value.symlink.slice(buf); + + if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { + cache_dir_subpath = "."; + cache_dir = std.fs.cwd(); + } else { + const global_link_dir = manager.globalLinkDirPath() catch unreachable; + var ptr = folder_path_buf; + var remain: []u8 = folder_path_buf[0..]; + @memcpy(ptr[0..global_link_dir.len], global_link_dir); + remain = remain[global_link_dir.len..]; + if (global_link_dir[global_link_dir.len - 1] != std.fs.path.sep) { + remain[0] = std.fs.path.sep; + remain = remain[1..]; + } + @memcpy(remain[0..folder.len], folder); + remain = remain[folder.len..]; + remain[0] = 0; + const len = @intFromPtr(remain.ptr) - @intFromPtr(ptr); + cache_dir_subpath = folder_path_buf[0..len :0]; + cache_dir = directory; + } }, + else => {}, } return .{ @@ -10153,6 +10126,7 @@ pub const PackageManager = struct { pathbuf: *bun.PathBuffer, comptime log_level: Options.LogLevel, ) !?PatchCommitResult { + var folder_path_buf: bun.PathBuffer = undefined; var lockfile: *Lockfile = try manager.allocator.create(Lockfile); defer lockfile.deinit(); switch (lockfile.loadFromDisk(manager, manager.allocator, manager.log, manager.options.lockfile_path, true)) { @@ -10275,7 +10249,12 @@ pub const PackageManager = struct { }, }; - const cache_result = manager.computeCacheDirAndSubpath(name, &actual_package.resolution, null); + const cache_result = manager.computeCacheDirAndSubpath( + name, + &actual_package.resolution, + &folder_path_buf, + null, + ); const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; @@ -10310,7 +10289,12 @@ pub const PackageManager = struct { name, }, .auto); const pkg = lockfile.packages.get(pkg_id); - const cache_result = manager.computeCacheDirAndSubpath(pkg.name.slice(lockfile.buffers.string_bytes.items), &pkg.resolution, null); + const cache_result = manager.computeCacheDirAndSubpath( + pkg.name.slice(lockfile.buffers.string_bytes.items), + &pkg.resolution, + &folder_path_buf, + null, + ); const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; break :brk .{ cache_dir, cache_dir_subpath, changes_dir, pkg }; @@ -10482,272 +10466,6 @@ pub const PackageManager = struct { }; } - /// - Arg is the tempdir containing the package with changes - /// - Get the patch file contents by running git diff on the temp dir and the original package dir - /// - Write the patch file to $PATCHES_DIR/$PKG_NAME_AND_VERSION.patch - /// - Update "patchedDependencies" in package.json - /// - Run install to install newly patched pkg - fn doPatchCommitOld( - manager: *PackageManager, - pathbuf: *bun.PathBuffer, - comptime log_level: Options.LogLevel, - ) !PatchCommitResult { - var lockfile: *Lockfile = try manager.allocator.create(Lockfile); - defer lockfile.deinit(); - switch (lockfile.loadFromDisk(manager, manager.allocator, manager.log, manager.options.lockfile_path, true)) { - .not_found => { - Output.panic("Lockfile not found", .{}); - }, - .err => |cause| { - if (log_level != .silent) { - switch (cause.step) { - .open_file => Output.prettyError("error opening lockfile: {s}\n", .{ - @errorName(cause.value), - }), - .parse_file => Output.prettyError("error parsing lockfile: {s}\n", .{ - @errorName(cause.value), - }), - .read_file => Output.prettyError("error reading lockfile: {s}\n", .{ - @errorName(cause.value), - }), - .migrating => Output.prettyError("error migrating lockfile: {s}\n", .{ - @errorName(cause.value), - }), - } - - if (manager.options.enable.fail_early) { - Output.prettyError("failed to load lockfile\n", .{}); - } else { - Output.prettyError("ignoring lockfile\n", .{}); - } - - Output.flush(); - } - Global.crash(); - }, - .ok => {}, - } - - const patched_pkg_folder = manager.options.positionals[1]; - if (patched_pkg_folder.len >= bun.MAX_PATH_BYTES) { - Output.prettyError("error: argument provided is too long\n", .{}); - Output.flush(); - Global.crash(); - } - @memcpy(pathbuf[0..patched_pkg_folder.len], patched_pkg_folder); - pathbuf[patched_pkg_folder.len] = 0; - - var versionbuf: [1024]u8 = undefined; - const version = switch (patchCommitGetVersion( - &versionbuf, - bun.path.joinZ(&[_][]const u8{ patched_pkg_folder, ".bun-patch-tag" }, .auto), - )) { - .result => |v| v, - .err => |e| { - Output.prettyError("error: failed to get bun patch tag: {}\n", .{e.toSystemError()}); - Output.flush(); - Global.crash(); - }, - }; - - const package_json_source: logger.Source = brk: { - const patched_pkg_folderZ = pathbuf[0..patched_pkg_folder.len :0]; - const pkgjsonpath = bun.path.joinZ(&[_][]const u8{ - patched_pkg_folderZ, - "package.json", - }, .auto); - - switch (bun.sys.File.toSource(pkgjsonpath, manager.allocator)) { - .result => |s| break :brk s, - .err => |e| { - Output.prettyError( - "error: failed to read package.json: {}\n", - .{e.withPath(pkgjsonpath).toSystemError()}, - ); - Output.flush(); - Global.crash(); - }, - } - }; - defer manager.allocator.free(package_json_source.contents); - - var package = Lockfile.Package{}; - try package.parse(lockfile, manager.allocator, manager.log, package_json_source, void, {}, Features.folder); - const name = lockfile.str(&package.name); - var resolution_buf: [1024]u8 = undefined; - const actual_package = switch (lockfile.package_index.get(package.name_hash) orelse { - Output.prettyError( - "error: failed to find package in lockfile package index, this is a bug in Bun. Please file a GitHub issue.\n", - .{}, - ); - Output.flush(); - Global.crash(); - }) { - .PackageID => |id| lockfile.packages.get(id), - .PackageIDMultiple => |ids| brk: { - for (ids.items) |id| { - const pkg = lockfile.packages.get(id); - const resolution_label = std.fmt.bufPrint(&resolution_buf, "{}", .{pkg.resolution.fmt(lockfile.buffers.string_bytes.items, .posix)}) catch unreachable; - if (std.mem.eql(u8, resolution_label, version)) { - break :brk pkg; - } - } - Output.prettyError("error: could not find package with name: {s}\n", .{ - package.name.slice(lockfile.buffers.string_bytes.items), - }); - Output.flush(); - Global.crash(); - }, - }; - const resolution_label = std.fmt.bufPrint(&resolution_buf, "{s}@{}", .{ name, actual_package.resolution.fmt(lockfile.buffers.string_bytes.items, .posix) }) catch unreachable; - const stuff = manager.computeCacheDirAndSubpath(name, &actual_package.resolution, null); - - const patchfile_contents = brk: { - const new_folder = patched_pkg_folder; - var buf2: bun.PathBuffer = undefined; - const old_folder = old_folder: { - const cache_dir_path = switch (bun.sys.getFdPath(bun.toFD(stuff.cache_dir.fd), &buf2)) { - .result => |s| s, - .err => |e| { - Output.prettyError( - "error: failed to read from cache {}\n", - .{e.toSystemError()}, - ); - Output.flush(); - Global.crash(); - }, - }; - break :old_folder bun.path.join(&[_][]const u8{ - cache_dir_path, - stuff.cache_dir_subpath, - }, .posix); - }; - break :brk switch (bun.patch.gitDiff(manager.allocator, old_folder, new_folder) catch |e| { - Output.prettyError( - "error: failed to make diff {s}\n", - .{@errorName(e)}, - ); - Output.flush(); - Global.crash(); - }) { - .result => |stdout| stdout, - .err => |stderr| { - defer stderr.deinit(); - const Truncate = struct { - stderr: std.ArrayList(u8), - - pub fn format( - this: *const @This(), - comptime _: []const u8, - _: std.fmt.FormatOptions, - writer: anytype, - ) !void { - const truncate_stderr = this.stderr.items.len > 256; - if (truncate_stderr) { - try writer.print("{s}... ({d} more bytes)", .{ this.stderr.items[0..256], this.stderr.items.len - 256 }); - } else try writer.print("{s}", .{this.stderr.items[0..]}); - } - }; - Output.prettyError( - "error: failed to make diff {}\n", - .{ - Truncate{ .stderr = stderr }, - }, - ); - Output.flush(); - Global.crash(); - }, - }; - }; - defer patchfile_contents.deinit(); - - // write the patch contents to temp file then rename - var tmpname_buf: [1024]u8 = undefined; - const tempfile_name = bun.span(try bun.fs.FileSystem.instance.tmpname("tmp", &tmpname_buf, bun.fastRandom())); - const tmpdir = try bun.fs.FileSystem.instance.tmpdir(); - const tmpfd = switch (bun.sys.openat( - bun.toFD(tmpdir.fd), - tempfile_name, - std.os.O.RDWR | std.os.O.CREAT, - 0o666, - )) { - .result => |fd| fd, - .err => |e| { - Output.prettyError( - "error: failed to open temp file {}\n", - .{e.toSystemError()}, - ); - Output.flush(); - Global.crash(); - }, - }; - defer _ = bun.sys.close(tmpfd); - - if (bun.sys.File.writeAll(.{ .handle = tmpfd }, patchfile_contents.items).asErr()) |e| { - Output.prettyError( - "error: failed to write patch to temp file {}\n", - .{e.toSystemError()}, - ); - Output.flush(); - Global.crash(); - } - - @memcpy(resolution_buf[resolution_label.len .. resolution_label.len + ".patch".len], ".patch"); - var patch_filename: []const u8 = resolution_buf[0 .. resolution_label.len + ".patch".len]; - var deinit = false; - if (escapePatchFilename(manager.allocator, patch_filename)) |escaped| { - deinit = true; - patch_filename = escaped; - } - defer if (deinit) manager.allocator.free(patch_filename); - - const path_in_patches_dir = bun.path.joinZ( - &[_][]const u8{ - manager.options.patch_features.commit.patches_dir, - patch_filename, - }, - .posix, - ); - - var nodefs = bun.JSC.Node.NodeFS{}; - const args = bun.JSC.Node.Arguments.Mkdir{ - .path = .{ .string = bun.PathString.init(manager.options.patch_features.commit.patches_dir) }, - }; - if (nodefs.mkdirRecursive(args, .sync).asErr()) |e| { - Output.prettyError( - "error: failed to make patches dir {}\n", - .{e.toSystemError()}, - ); - Output.flush(); - Global.crash(); - } - - // rename to patches dir - - if (bun.sys.renameatConcurrently( - bun.toFD(tmpdir.fd), - tempfile_name, - bun.FD.cwd(), - path_in_patches_dir, - ).asErr()) |e| { - Output.prettyError( - "error: failed to renaming patch file to patches dir {}\n", - .{e.toSystemError()}, - ); - Output.flush(); - Global.crash(); - } - - const patch_key = std.fmt.allocPrint(manager.allocator, "{s}", .{resolution_label}) catch bun.outOfMemory(); - const patchfile_path = manager.allocator.dupe(u8, path_in_patches_dir) catch bun.outOfMemory(); - _ = bun.sys.unlink(bun.path.joinZ(&[_][]const u8{ patched_pkg_folder, ".bun-patch-tag" }, .auto)); - - return .{ - .patch_key = patch_key, - .patchfile_path = patchfile_path, - }; - } - fn patchCommitGetVersion( buf: *[1024]u8, patch_tag_path: [:0]const u8, diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index cf50e287199f5c..3686ef021aa4ec 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -496,9 +496,11 @@ pub const PatchTask = struct { const pkg_name = pkg_manager.lockfile.packages.items(.name)[pkg_id]; const resolution: *const Resolution = &pkg_manager.lockfile.packages.items(.resolution)[pkg_id]; + var folder_path_buf: bun.PathBuffer = undefined; const stuff = pkg_manager.computeCacheDirAndSubpath( pkg_name.slice(pkg_manager.lockfile.buffers.string_bytes.items), resolution, + &folder_path_buf, patch_hash, ); From faacbc9159f328274591fced0dd9c673cfb8305d Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:56:34 -0700 Subject: [PATCH 06/51] stuff --- src/bun.js/bindings/bindings.zig | 17 ++ src/install/install.zig | 255 +++++++++++++++++++++++++- test/cli/install/bun-patch.test.ts | 282 +++++++++++++++++++++++++++++ 3 files changed, 544 insertions(+), 10 deletions(-) create mode 100644 test/cli/install/bun-patch.test.ts diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 466dd621f9f8a6..ddf8f00d265a79 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -1646,6 +1646,23 @@ pub const SystemError = extern struct { } pub fn format(self: SystemError, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + if (self.path) { + // TODO: remove this hardcoding + switch (bun.Output.enable_ansi_colors_stderr) { + inline else => |enable_colors| try writer.print( + comptime bun.Output.prettyFmt( + "{}: {s}:{} ({}())", + enable_colors, + ), + .{ + self.code, + self.path, + self.message, + self.syscall, + }, + ), + } + } else // TODO: remove this hardcoding switch (bun.Output.enable_ansi_colors_stderr) { inline else => |enable_colors| try writer.print( diff --git a/src/install/install.zig b/src/install/install.zig index 93caaf9295e263..1e43b79355e564 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -9635,7 +9635,10 @@ pub const PackageManager = struct { } } - const updates: []UpdateRequest = if (manager.subcommand == .@"patch-commit") &[_]UpdateRequest{} else UpdateRequest.parse(ctx.allocator, ctx.log, manager.options.positionals[1..], &update_requests, manager.subcommand); + const updates: []UpdateRequest = if (manager.subcommand == .@"patch-commit" or manager.subcommand == .patch) + &[_]UpdateRequest{} + else + UpdateRequest.parse(ctx.allocator, ctx.log, manager.options.positionals[1..], &update_requests, manager.subcommand); switch (manager.subcommand) { inline else => |subcommand| try manager.updatePackageJSONAndInstallWithManagerWithUpdates( ctx, @@ -10078,14 +10081,13 @@ pub const PackageManager = struct { /// - Give that to user fn preparePatch(manager: *PackageManager) !void { const strbuf = manager.lockfile.buffers.string_bytes.items; - _ = strbuf; // autofix const pkg_maybe_version_to_patch = manager.options.positionals[1]; const name: []const u8, const version: ?[]const u8 = brk: { if (std.mem.indexOfScalar(u8, pkg_maybe_version_to_patch[1..], '@')) |version_delimiter| { break :brk .{ - pkg_maybe_version_to_patch[0..version_delimiter], - pkg_maybe_version_to_patch[1..][version_delimiter + 1 ..], + pkg_maybe_version_to_patch[0 .. version_delimiter + 1], + pkg_maybe_version_to_patch[version_delimiter + 2 ..], }; } break :brk .{ @@ -10096,21 +10098,185 @@ pub const PackageManager = struct { const result = pkg_dep_id_for_name_and_version(manager.lockfile, pkg_maybe_version_to_patch, name, version); const pkg_id = result[0]; - _ = pkg_id; // autofix const dependency_id = result[1]; var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); const folder = (try nodeModulesFolderForDependencyID(&iterator, dependency_id)) orelse { - @panic("TODO zack: bun patch node_modules folder not found"); + Output.prettyError( + "error: could not find the folder for {s} in node_modules\n", + .{pkg_maybe_version_to_patch}, + ); + Output.flush(); + Global.crash(); }; + var folder_path_buf: bun.PathBuffer = undefined; + + const pkg = manager.lockfile.packages.get(pkg_id); + const pkg_name = pkg.name.slice(strbuf); + const cache_result = manager.computeCacheDirAndSubpath(pkg_name, &pkg.resolution, &folder_path_buf, null); + + const cache_dir = cache_result.cache_dir; + const cache_dir_subpath = cache_result.cache_dir_subpath; + const module_folder = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); + + manager.overwriteNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { + Output.prettyError( + "error: error overwriting folder in node_modules: {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }; + Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ name, module_folder }); Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch-commit '{s}'\n", .{module_folder}); return; } + fn overwriteNodeModulesFolder( + manager: *PackageManager, + cache_dir: std.fs.Dir, + cache_dir_subpath: []const u8, + node_modules_folder_path: []const u8, + ) !void { + var node_modules_folder = try std.fs.cwd().openDir(node_modules_folder_path, .{ .iterate = true }); + defer node_modules_folder.close(); + + const IGNORED_PATHS: []const bun.OSPathSlice = &.{ + "node_modules", + ".git", + "CMakeFiles", + }; + + const FileCopier = struct { + pub fn copy( + destination_dir_: std.fs.Dir, + walker: *Walker, + to_copy_into1: if (Environment.isWindows) []u16 else void, + head1: if (Environment.isWindows) []u16 else void, + to_copy_into2: if (Environment.isWindows) []u16 else void, + head2: if (Environment.isWindows) []u16 else void, + ) !u32 { + var real_file_count: u32 = 0; + + var copy_file_state: bun.CopyFileState = .{}; + + while (try walker.next()) |entry| { + if (comptime Environment.isWindows) { + switch (entry.kind) { + .directory, .file => {}, + else => continue, + } + + if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { + return error.NameTooLong; + } + + @memcpy(to_copy_into1[0..entry.path.len], entry.path); + head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; + const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; + + @memcpy(to_copy_into2[0..entry.path.len], entry.path); + head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; + const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; + + switch (entry.kind) { + .directory => { + if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { + bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; + } + }, + .file => { + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) == 0) { + if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { + bun.MakePath.makePath(u16, destination_dir_, entry_dirname) catch {}; + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) != 0) { + continue; + } + } + + if (bun.windows.Win32Error.get().toSystemErrno()) |err| { + Output.prettyError("{s}: copying file {}", .{ @tagName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + } else { + Output.prettyError("error copying file {}", .{bun.fmt.fmtOSPath(entry.path, .{})}); + } + + Global.crash(); + } + }, + else => unreachable, // handled above + } + } else { + if (entry.kind != .file) continue; + real_file_count += 1; + const openFile = std.fs.Dir.openFile; + const createFile = std.fs.Dir.createFile; + + var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); + defer in_file.close(); + + if (bun.sys.unlinkat( + bun.toFD(destination_dir_.fd), + entry.basename.ptr[0..entry.basename.len :0], + ).asErr()) |e| { + std.debug.print("unlink problem uh oh {}\n", .{e}); + @panic("TODO zack handle"); + } + + const mode = try in_file.mode(); + var outfile = try createFile(destination_dir_, entry.basename, .{ .mode = mode }); + defer outfile.close(); + + debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); + // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { + // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { + // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; + // } + // break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { + // if (do_progress) { + // progress_.root.end(); + // progress_.refresh(); + // } + + // Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + // Global.crash(); + // }; + // }; + // defer outfile.close(); + + if (comptime Environment.isPosix) { + const stat = in_file.stat() catch continue; + _ = C.fchmod(outfile.handle, @intCast(stat.mode)); + } + + bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; + } + } + + return real_file_count; + } + }; + + var pkg_in_cache_dir = try cache_dir.openDir(cache_dir_subpath, .{ .iterate = true }); + defer pkg_in_cache_dir.close(); + var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); + defer walker.deinit(); + _ = try FileCopier.copy( + node_modules_folder, + &walker, + {}, + {}, + {}, + {}, + ); + } + const PatchCommitResult = struct { patch_key: []const u8, patchfile_path: []const u8, @@ -10176,6 +10342,19 @@ pub const PackageManager = struct { break :brk .name_and_version; }; + // Attempt to open the existing node_modules folder + const root_node_modules = switch (bun.sys.openatOSPath(bun.FD.cwd(), bun.OSPathLiteral("node_modules"), std.os.O.DIRECTORY | std.os.O.RDONLY, 0o755)) { + .result => |fd| std.fs.Dir{ .fd = fd.cast() }, + .err => |e| { + Output.prettyError( + "error: failed to open root node_modules folder: {}\n", + .{e}, + ); + Output.flush(); + Global.crash(); + }, + }; + var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); var resolution_buf: [1024]u8 = undefined; const _cache_dir: std.fs.Dir, const _cache_dir_subpath: stringZ, const _changes_dir: []const u8, const _pkg: Package = switch (arg_kind) { @@ -10282,7 +10461,12 @@ pub const PackageManager = struct { &iterator, dependency_id, )) orelse { - @panic("TODO zack node_modules not found"); + Output.prettyError( + "error: could not find the folder for {s} in node_modules\n", + .{argument}, + ); + Output.flush(); + Global.crash(); }; const changes_dir = bun.path.joinZBuf(pathbuf[0..], &[_][]const u8{ node_modules.relative_path, @@ -10330,6 +10514,57 @@ pub const PackageManager = struct { cache_dir_subpath, }, .posix); }; + + const random_tempdir = bun.span(bun.fs.FileSystem.instance.tmpname(name, buf2[0..], bun.fastRandom()) catch |e| { + Output.prettyError( + "error: failed to make tempdir {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }); + const has_nested_node_modules = has_nested_node_modules: { + var new_folder_handle = std.fs.cwd().openDir(new_folder, .{}) catch |e| { + Output.prettyError( + "error: failed to open directory {s} {s}\n", + .{ new_folder, @errorName(e) }, + ); + Output.flush(); + Global.crash(); + }; + defer new_folder_handle.close(); + + if (bun.sys.renameatConcurrently( + bun.toFD(new_folder_handle.fd), + "node_modules", + bun.toFD(root_node_modules.fd), + random_tempdir, + ).asErr()) |_| break :has_nested_node_modules false; + + break :has_nested_node_modules true; + }; + defer { + if (has_nested_node_modules) { + var new_folder_handle = std.fs.cwd().openDir(new_folder, .{}) catch |e| { + Output.prettyError( + "error: failed to open directory {s} {s}\n", + .{ new_folder, @errorName(e) }, + ); + Output.flush(); + Global.crash(); + }; + defer new_folder_handle.close(); + if (bun.sys.renameatConcurrently( + bun.toFD(root_node_modules.fd), + random_tempdir, + bun.toFD(new_folder_handle.fd), + "node_modules", + ).asErr()) |e| { + Output.warn("failed renaming nested node_modules folder, this may cause issues: {}", .{e}); + } + } + } + std.debug.print("OLD FOLDER: {s}\n", .{old_folder}); std.debug.print("NEW FOLDER: {s}\n", .{new_folder}); const contents = switch (bun.patch.gitDiff(manager.allocator, old_folder, new_folder) catch |e| { @@ -11072,9 +11307,9 @@ pub const PackageManager = struct { break :brk ""; } else std.fmt.bufPrint(&resolution_buf, "{}", .{resolution.fmt(buf, .posix)}) catch unreachable; - if (std.mem.eql(u8, "is-even", name)) { - std.debug.print("HELLO!\n", .{}); - } + // if (std.mem.eql(u8, "is-even", name)) { + // std.debug.print("HELLO!\n", .{}); + // } const patch_patch, const patch_contents_hash, const patch_name_and_version_hash, const remove_patch = brk: { if (this.manager.lockfile.patched_dependencies.entries.len == 0 and this.manager.patched_dependencies_to_remove.entries.len == 0) break :brk .{ null, null, null, false }; diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts new file mode 100644 index 00000000000000..690e26c77f7532 --- /dev/null +++ b/test/cli/install/bun-patch.test.ts @@ -0,0 +1,282 @@ +import { $, ShellOutput, ShellPromise } from "bun"; +import { bunExe, bunEnv as env, toBeValidBin, toHaveBins, toBeWorkspaceLink, tempDirWithFiles, bunEnv } from "harness"; +import { afterAll, afterEach, beforeAll, beforeEach, expect, it, describe, test, setDefaultTimeout } from "bun:test"; +import { join, sep } from "path"; + +describe("bun patch ", async () => { + test("should patch a package when it is already patched", async () => {}); + + test("bad patch arg", async () => { + const tempdir = tempDirWithFiles("lol", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); + + await $`${bunExe()} i`.env(bunEnv).cwd(tempdir); + const { stderr, exitCode } = await $`${bunExe()} patch lkflksdkfj`.env(bunEnv).cwd(tempdir).throws(false); + expect(exitCode).toBe(1); + expect(stderr.toString()).toContain("error: package lkflksdkfj not found"); + }); + + test("bad patch commit arg", async () => { + const tempdir = tempDirWithFiles("lol", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); + + await $`${bunExe()} i`.env(bunEnv).cwd(tempdir); + const { stderr } = await $`${bunExe()} patch is-even`.env(bunEnv).cwd(tempdir); + expect(stderr.toString()).not.toContain("error"); + + const { stderr: stderr2 } = await $`${bunExe()} patch-commit lskfjdslkfjsldkfjlsdkfj`.env(bunEnv).cwd(tempdir); + expect(stderr.toString()).not.toContain("error"); + }); + + function makeTest( + name: string, + { + dependencies, + mainScript, + patchArg, + patchedCode, + expected, + }: { + dependencies: Record; + mainScript: string; + patchArg: string; + patchedCode: string; + expected: { patchName: string; patchPath: string; stdout: string }; + extra?: (filedir: string) => Promise; + }, + ) { + test(name, async () => { + $.throws(true); + + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": dependencies, + }), + "index.ts": mainScript, + }); + + { + const { stderr } = await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } + + { + const { stderr, stdout } = await $`${bunExe()} patch ${patchArg}`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + expect(stdout.toString()).toContain( + `To patch ${expected.patchName}, edit the following folder: + + ${expected.patchPath} + +Once you're done with your changes, run: + + bun patch-commit '${expected.patchPath}'`, + ); + } + + { + const newCode = patchedCode; + + await $`echo ${newCode} > ${expected.patchPath}/index.js`.env(bunEnv).cwd(filedir); + const { stderr, stdout } = await $`${bunExe()} patch-commit ${expected.patchPath}`.env(bunEnv).cwd(filedir); + } + + const output = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(filedir).text(); + expect(output).toBe(expected.stdout); + }); + } + + ["is-even@1.0.0"].map(patchArg => + makeTest("should patch a node_modules package", { + dependencies: { "is-even": "1.0.0" }, + mainScript: /* ts */ `import isEven from 'is-even'; isEven(420)`, + patchArg, + patchedCode: /* ts */ `/*! + * is-even + * + * Copyright (c) 2015, 2017, Jon Schlinkert. + * Released under the MIT License. + */ + + 'use strict'; + + var isOdd = require('is-odd'); + + module.exports = function isEven(i) { + console.log("If you're reading this, the patch worked!") + return !isOdd(i); + }; + `, + expected: { + patchName: "is-even", + patchPath: "node_modules/is-even", + stdout: "If you're reading this, the patch worked!\n", + }, + }), + ); + + ["is-odd@0.1.2"].map(patchArg => + makeTest("should patch a nested node_modules package", { + dependencies: { "is-even": "1.0.0", "is-odd": "3.0.1" }, + mainScript: /* ts */ `import isEven from 'is-even'; isEven(420)`, + patchArg, + patchedCode: /* ts */ `/*! + * is-odd + * + * Copyright (c) 2015-2017, Jon Schlinkert. + * Released under the MIT License. + */ + + 'use strict'; + + var isNumber = require('is-number'); + + module.exports = function isOdd(i) { + if (!isNumber(i)) { + throw new TypeError('is-odd expects a number.'); + } + if (Number(i) !== Math.floor(i)) { + throw new RangeError('is-odd expects an integer.'); + } + console.log("If you're reading this, the patch worked.") + return !!(~~i & 1); + }; + `, + expected: { + patchName: "is-odd", + patchPath: "node_modules/is-even/node_modules/is-odd", + stdout: "If you're reading this, the patch worked.\n", + }, + extra: async filedir => { + const patchfile = await $`cat ${join(filedir, "patches", "is-odd@0.1.2.patch")}`.cwd(filedir).text(); + // ensure node modules is not in the patch + expect(patchfile).not.toContain("node_modules"); + }, + }), + ); + + test("should overwrite the node_modules folder of the package", async () => { + $.throws(true); + + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); + + { + const { stderr } = await $`${bunExe()} i --backend hardlink`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).toContain("Saved lockfile"); + + const newCode = /* ts */ ` +module.exports = function isEven() { + return 'LOL' +} +`; + + await $`${bunExe()} patch is-even`.env(bunEnv).cwd(filedir); + await $`echo ${newCode} > node_modules/is-even/index.js`.env(bunEnv).cwd(filedir); + } + + const tempdir = tempDirWithFiles("unpatched", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); + + await $`${bunExe()} run index.ts` + .env(bunEnv) + .cwd(filedir) + .then(o => expect(o.stderr.toString()).toBe("")); + + const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toBe("true\n"); + }); + + test("should overwrite nested node_modules folder of the package", async () => { + $.throws(true); + + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + "is-odd": "3.0.1", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); + + { + const { stderr } = await $`${bunExe()} i --backend hardlink`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).toContain("Saved lockfile"); + + const newCode = /* ts */ ` +module.exports = function isOdd() { + return 'LOL' +} +`; + + await $`ls -d node_modules/is-even/node_modules/is-odd`.cwd(filedir); + await $`${bunExe()} patch is-odd@0.1.2`.env(bunEnv).cwd(filedir); + await $`echo ${newCode} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(filedir); + } + + const tempdir = tempDirWithFiles("unpatched", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); + + await $`${bunExe()} run index.ts` + .env(bunEnv) + .cwd(filedir) + .then(o => expect(o.stderr.toString()).toBe("")); + + const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toBe("true\n"); + }); +}); From a2dc15e9a5c753ca9045337b80219ca67ba89842 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:22:27 -0700 Subject: [PATCH 07/51] more stuff --- src/bun.js/bindings/bindings.zig | 4 +- src/install/install.zig | 184 +++++++++++++++++++++++------ test/cli/install/bun-patch.test.ts | 171 ++++++++++++++------------- 3 files changed, 241 insertions(+), 118 deletions(-) diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index ddf8f00d265a79..348435a145ee30 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -1646,12 +1646,12 @@ pub const SystemError = extern struct { } pub fn format(self: SystemError, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { - if (self.path) { + if (!self.path.isEmpty()) { // TODO: remove this hardcoding switch (bun.Output.enable_ansi_colors_stderr) { inline else => |enable_colors| try writer.print( comptime bun.Output.prettyFmt( - "{}: {s}:{} ({}())", + "{}: {s}: {} ({}())", enable_colors, ), .{ diff --git a/src/install/install.zig b/src/install/install.zig index 1e43b79355e564..f617e4efb75c43 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10075,51 +10075,163 @@ pub const PackageManager = struct { return .{ pkg_id, dependency_id }; } - /// - Arg is name and possibly version (e.g. "is-even" or "is-even@1.0.0") - /// - Find package that satisfies name and version - /// - Copy contents of package into temp dir - /// - Give that to user + /// 1. Arg is either: + /// - name and possibly version (e.g. "is-even" or "is-even@1.0.0") + /// - path to package in node_modules + /// 2. Calculate cache dir for package + /// 3. Overwrite the input package with the one from the cache (cuz it could be hardlinked) + /// 4. Print to user fn preparePatch(manager: *PackageManager) !void { const strbuf = manager.lockfile.buffers.string_bytes.items; - const pkg_maybe_version_to_patch = manager.options.positionals[1]; + const argument = manager.options.positionals[1]; - const name: []const u8, const version: ?[]const u8 = brk: { - if (std.mem.indexOfScalar(u8, pkg_maybe_version_to_patch[1..], '@')) |version_delimiter| { - break :brk .{ - pkg_maybe_version_to_patch[0 .. version_delimiter + 1], - pkg_maybe_version_to_patch[version_delimiter + 2 ..], - }; - } - break :brk .{ - pkg_maybe_version_to_patch, - null, - }; + const ArgKind = enum { + path, + name_and_version, }; - const result = pkg_dep_id_for_name_and_version(manager.lockfile, pkg_maybe_version_to_patch, name, version); - const pkg_id = result[0]; - const dependency_id = result[1]; - - var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); - const folder = (try nodeModulesFolderForDependencyID(&iterator, dependency_id)) orelse { - Output.prettyError( - "error: could not find the folder for {s} in node_modules\n", - .{pkg_maybe_version_to_patch}, - ); - Output.flush(); - Global.crash(); + const arg_kind: ArgKind = brk: { + if (bun.strings.hasPrefix(argument, "node_modules/")) break :brk .path; + if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules/")) break :brk .path; + break :brk .name_and_version; }; var folder_path_buf: bun.PathBuffer = undefined; + var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); + var resolution_buf: [1024]u8 = undefined; + + const cache_dir: std.fs.Dir, const cache_dir_subpath: []const u8, const module_folder: []const u8, const pkg_name: []const u8 = switch (arg_kind) { + .path => brk: { + var lockfile = manager.lockfile; + const package_json_source: logger.Source = src: { + const package_json_path = bun.path.joinZ(&[_][]const u8{ argument, "package.json" }, .auto); + + switch (bun.sys.File.toSource(package_json_path, manager.allocator)) { + .result => |s| break :src s, + .err => |e| { + Output.prettyError( + "error: failed to read package.json: {}\n", + .{e.withPath(package_json_path).toSystemError()}, + ); + Output.flush(); + Global.crash(); + }, + } + }; + defer manager.allocator.free(package_json_source.contents); + + initializeStore(); + const json = json_parser.ParsePackageJSONUTF8AlwaysDecode(&package_json_source, manager.log, manager.allocator) catch |err| { + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; + }, + } + Output.prettyErrorln("{s} parsing package.json in \"{s}\"", .{ @errorName(err), package_json_source.path.prettyDir() }); + Global.crash(); + }; + + const version = version: { + if (json.asProperty("version")) |v| { + if (v.expr.asString(manager.allocator)) |s| break :version s; + } + Output.prettyError( + "error: invalid package.json, missing or invalid property \"version\": {s}\n", + .{package_json_source.path.text}, + ); + Output.flush(); + Global.crash(); + }; + + var package = Lockfile.Package{}; + try package.parseWithJSON(lockfile, manager.allocator, manager.log, package_json_source, json, void, {}, Features.folder); - const pkg = manager.lockfile.packages.get(pkg_id); - const pkg_name = pkg.name.slice(strbuf); - const cache_result = manager.computeCacheDirAndSubpath(pkg_name, &pkg.resolution, &folder_path_buf, null); + const name = lockfile.str(&package.name); + const actual_package = switch (lockfile.package_index.get(package.name_hash) orelse { + Output.prettyError( + "error: failed to find package in lockfile package index, this is a bug in Bun. Please file a GitHub issue.\n", + .{}, + ); + Output.flush(); + Global.crash(); + }) { + .PackageID => |id| lockfile.packages.get(id), + .PackageIDMultiple => |ids| id: { + for (ids.items) |id| { + const pkg = lockfile.packages.get(id); + const resolution_label = std.fmt.bufPrint(&resolution_buf, "{}", .{pkg.resolution.fmt(lockfile.buffers.string_bytes.items, .posix)}) catch unreachable; + if (std.mem.eql(u8, resolution_label, version)) { + break :id pkg; + } + } + Output.prettyError("error: could not find package with name: {s}\n", .{ + package.name.slice(lockfile.buffers.string_bytes.items), + }); + Output.flush(); + Global.crash(); + }, + }; - const cache_dir = cache_result.cache_dir; - const cache_dir_subpath = cache_result.cache_dir_subpath; + const cache_result = manager.computeCacheDirAndSubpath( + name, + &actual_package.resolution, + &folder_path_buf, + null, + ); + const cache_dir = cache_result.cache_dir; + const cache_dir_subpath = cache_result.cache_dir_subpath; - const module_folder = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); + break :brk .{ + cache_dir, + cache_dir_subpath, + argument, + name, + }; + }, + .name_and_version => brk: { + const pkg_maybe_version_to_patch = argument; + const name: []const u8, const version: ?[]const u8 = namever: { + if (std.mem.indexOfScalar(u8, pkg_maybe_version_to_patch[1..], '@')) |version_delimiter| { + break :namever .{ + pkg_maybe_version_to_patch[0 .. version_delimiter + 1], + pkg_maybe_version_to_patch[version_delimiter + 2 ..], + }; + } + break :namever .{ + pkg_maybe_version_to_patch, + null, + }; + }; + + const result = pkg_dep_id_for_name_and_version(manager.lockfile, pkg_maybe_version_to_patch, name, version); + const pkg_id = result[0]; + const dependency_id = result[1]; + + const folder = (try nodeModulesFolderForDependencyID(&iterator, dependency_id)) orelse { + Output.prettyError( + "error: could not find the folder for {s} in node_modules\n", + .{pkg_maybe_version_to_patch}, + ); + Output.flush(); + Global.crash(); + }; + + const pkg = manager.lockfile.packages.get(pkg_id); + const pkg_name = pkg.name.slice(strbuf); + const cache_result = manager.computeCacheDirAndSubpath(pkg_name, &pkg.resolution, &folder_path_buf, null); + + const cache_dir = cache_result.cache_dir; + const cache_dir_subpath = cache_result.cache_dir_subpath; + + const module_folder = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); + break :brk .{ + cache_dir, + cache_dir_subpath, + module_folder, + pkg_name, + }; + }, + }; manager.overwriteNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { Output.prettyError( @@ -10130,7 +10242,7 @@ pub const PackageManager = struct { Global.crash(); }; - Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ name, module_folder }); + Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ pkg_name, module_folder }); Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch-commit '{s}'\n", .{module_folder}); return; @@ -10355,7 +10467,7 @@ pub const PackageManager = struct { }, }; - var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); + var iterator = Lockfile.Tree.Iterator.init(lockfile); var resolution_buf: [1024]u8 = undefined; const _cache_dir: std.fs.Dir, const _cache_dir_subpath: stringZ, const _changes_dir: []const u8, const _pkg: Package = switch (arg_kind) { .path => result: { diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 690e26c77f7532..2e0a86bcf70873 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -42,8 +42,11 @@ describe("bun patch ", async () => { const { stderr } = await $`${bunExe()} patch is-even`.env(bunEnv).cwd(tempdir); expect(stderr.toString()).not.toContain("error"); - const { stderr: stderr2 } = await $`${bunExe()} patch-commit lskfjdslkfjsldkfjlsdkfj`.env(bunEnv).cwd(tempdir); - expect(stderr.toString()).not.toContain("error"); + const { stderr: stderr2 } = await $`${bunExe()} patch-commit lskfjdslkfjsldkfjlsdkfj` + .env(bunEnv) + .cwd(tempdir) + .throws(false); + expect(stderr2.toString()).toContain("error: package lskfjdslkfjsldkfjlsdkfj not found"); }); function makeTest( @@ -107,7 +110,7 @@ Once you're done with your changes, run: }); } - ["is-even@1.0.0"].map(patchArg => + ["is-even@1.0.0", "node_modules/is-even"].map(patchArg => makeTest("should patch a node_modules package", { dependencies: { "is-even": "1.0.0" }, mainScript: /* ts */ `import isEven from 'is-even'; isEven(420)`, @@ -136,7 +139,7 @@ Once you're done with your changes, run: }), ); - ["is-odd@0.1.2"].map(patchArg => + ["is-odd@0.1.2", "node_modules/is-even/node_modules/is-odd"].map(patchArg => makeTest("should patch a nested node_modules package", { dependencies: { "is-even": "1.0.0", "is-odd": "3.0.1" }, mainScript: /* ts */ `import isEven from 'is-even'; isEven(420)`, @@ -177,106 +180,114 @@ Once you're done with your changes, run: ); test("should overwrite the node_modules folder of the package", async () => { - $.throws(true); + const patchArgs = ["is-even@1.0.0", "node_modules/is-even"]; - const filedir = tempDirWithFiles("patch1", { - "package.json": JSON.stringify({ - "name": "bun-patch-test", - "module": "index.ts", - "type": "module", - "dependencies": { - "is-even": "1.0.0", - }, - }), - "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, - }); + for (const patchArg of patchArgs) { + $.throws(true); - { - const { stderr } = await $`${bunExe()} i --backend hardlink`.env(bunEnv).cwd(filedir); - expect(stderr.toString()).toContain("Saved lockfile"); + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); - const newCode = /* ts */ ` + { + const { stderr } = await $`${bunExe()} i --backend hardlink`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).toContain("Saved lockfile"); + + const newCode = /* ts */ ` module.exports = function isEven() { return 'LOL' } `; - await $`${bunExe()} patch is-even`.env(bunEnv).cwd(filedir); - await $`echo ${newCode} > node_modules/is-even/index.js`.env(bunEnv).cwd(filedir); - } + await $`${bunExe()} patch ${patchArg}`.env(bunEnv).cwd(filedir); + await $`echo ${newCode} > node_modules/is-even/index.js`.env(bunEnv).cwd(filedir); + } - const tempdir = tempDirWithFiles("unpatched", { - "package.json": JSON.stringify({ - "name": "bun-patch-test", - "module": "index.ts", - "type": "module", - "dependencies": { - "is-even": "1.0.0", - }, - }), - "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, - }); + const tempdir = tempDirWithFiles("unpatched", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); - await $`${bunExe()} run index.ts` - .env(bunEnv) - .cwd(filedir) - .then(o => expect(o.stderr.toString()).toBe("")); + await $`${bunExe()} run index.ts` + .env(bunEnv) + .cwd(filedir) + .then(o => expect(o.stderr.toString()).toBe("")); - const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); - expect(stderr.toString()).toBe(""); - expect(stdout.toString()).toBe("true\n"); + const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toBe("true\n"); + } }); test("should overwrite nested node_modules folder of the package", async () => { - $.throws(true); + const patchArgs = ["is-odd@0.1.2", "node_modules/is-even/node_modules/is-odd"]; - const filedir = tempDirWithFiles("patch1", { - "package.json": JSON.stringify({ - "name": "bun-patch-test", - "module": "index.ts", - "type": "module", - "dependencies": { - "is-even": "1.0.0", - "is-odd": "3.0.1", - }, - }), - "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, - }); + for (const patchArg of patchArgs) { + $.throws(true); - { - const { stderr } = await $`${bunExe()} i --backend hardlink`.env(bunEnv).cwd(filedir); - expect(stderr.toString()).toContain("Saved lockfile"); + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + "is-odd": "3.0.1", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); + + { + const { stderr } = await $`${bunExe()} i --backend hardlink`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).toContain("Saved lockfile"); - const newCode = /* ts */ ` + const newCode = /* ts */ ` module.exports = function isOdd() { return 'LOL' } `; - await $`ls -d node_modules/is-even/node_modules/is-odd`.cwd(filedir); - await $`${bunExe()} patch is-odd@0.1.2`.env(bunEnv).cwd(filedir); - await $`echo ${newCode} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(filedir); - } + await $`ls -d node_modules/is-even/node_modules/is-odd`.cwd(filedir); + await $`${bunExe()} patch ${patchArg}`.env(bunEnv).cwd(filedir); + await $`echo ${newCode} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(filedir); + } - const tempdir = tempDirWithFiles("unpatched", { - "package.json": JSON.stringify({ - "name": "bun-patch-test", - "module": "index.ts", - "type": "module", - "dependencies": { - "is-even": "1.0.0", - }, - }), - "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, - }); + const tempdir = tempDirWithFiles("unpatched", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); - await $`${bunExe()} run index.ts` - .env(bunEnv) - .cwd(filedir) - .then(o => expect(o.stderr.toString()).toBe("")); + await $`${bunExe()} run index.ts` + .env(bunEnv) + .cwd(filedir) + .then(o => expect(o.stderr.toString()).toBe("")); - const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); - expect(stderr.toString()).toBe(""); - expect(stdout.toString()).toBe("true\n"); + const { stdout, stderr } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); + expect(stderr.toString()).toBe(""); + expect(stdout.toString()).toBe("true\n"); + } }); }); From 6e30a29b16f097651acab6f923115d86edfc8156 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Tue, 11 Jun 2024 19:57:15 -0700 Subject: [PATCH 08/51] make stuff work --- src/install/install.zig | 177 +++++++++++++++++++++++++---- src/install/patch_install.zig | 5 +- test/cli/install/bun-patch.test.ts | 88 +++++++++++++- 3 files changed, 244 insertions(+), 26 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index f617e4efb75c43..d6dd3f8b3419b5 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -69,6 +69,14 @@ pub const max_hex_hash_len: comptime_int = brk: { pub const max_buntag_hash_buf_len: comptime_int = max_hex_hash_len + bun_hash_tag.len + 1; pub const BuntagHashBuf = [max_buntag_hash_buf_len]u8; +pub fn buntaghashbuf_make(buf: *BuntagHashBuf, patch_hash: u64) [:0]u8 { + @memcpy(buf[0..bun_hash_tag.len], bun_hash_tag); + const digits = std.fmt.bufPrint(buf[bun_hash_tag.len..], "{x}", .{patch_hash}) catch bun.outOfMemory(); + buf[bun_hash_tag.len + digits.len] = 0; + const bunhashtag = buf[0 .. bun_hash_tag.len + digits.len :0]; + return bunhashtag; +} + pub const patch = @import("./patch_install.zig"); pub const PatchTask = patch.PatchTask; @@ -1037,9 +1045,7 @@ pub fn NewPackageInstall(comptime kind: PkgInstallKind) type { // hash from the .patch file, to be checked against bun tag const patchfile_contents_hash = this.patch.patch_contents_hash; var buf: BuntagHashBuf = undefined; - @memcpy(buf[0..bun_hash_tag.len], bun_hash_tag); - const digits = std.fmt.bufPrint(buf[bun_hash_tag.len..], "{x}", .{patchfile_contents_hash}) catch bun.outOfMemory(); - const bunhashtag = buf[0 .. bun_hash_tag.len + digits.len]; + const bunhashtag = buntaghashbuf_make(&buf, patchfile_contents_hash); const patch_tag_path = bun.path.joinZ(&[_][]const u8{ this.destination_dir_subpath, @@ -2652,6 +2658,9 @@ pub const PackageManager = struct { patch_calc_hash_batch: ThreadPool.Batch = .{}, patch_task_fifo: PatchTaskFifo = PatchTaskFifo.init(), patch_task_queue: PatchTaskQueue = .{}, + /// We actually need to calculate the patch file hashes + /// every single time, because someone could edit the patchfile at anytime + pending_pre_calc_hashes: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), pending_tasks: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), total_tasks: u32 = 0, preallocated_network_tasks: PreallocatedNetworkTasks = PreallocatedNetworkTasks.init(bun.default_allocator), @@ -4251,6 +4260,17 @@ pub const PackageManager = struct { this.patch_task_fifo.writeItemAssumeCapacity(task); } + pub fn enqueuePatchTaskPre(this: *PackageManager, task: *PatchTask) void { + debug("Enqueue patch task pre: 0x{x} {s}", .{ @intFromPtr(task), @tagName(task.callback) }); + task.pre = true; + if (this.patch_task_fifo.writableLength() == 0) { + this.flushPatchTaskQueue(); + } + + this.patch_task_fifo.writeItemAssumeCapacity(task); + _ = this.pending_pre_calc_hashes.fetchAdd(1, .Monotonic); + } + const SuccessFn = *const fn (*PackageManager, DependencyID, PackageID) void; const FailFn = *const fn (*PackageManager, *const Dependency, PackageID, anyerror) void; fn assignResolution(this: *PackageManager, dependency_id: DependencyID, package_id: PackageID) void { @@ -10049,8 +10069,8 @@ pub const PackageManager = struct { if (matches_found > 1) { Output.prettyErrorln( - "\nerror: please specify a precise version:", - .{}, + "\nerror: Found multiple versions of {s}, please specify a precise version from the following list:\n", + .{name}, ); var i: usize = 0; const pkg_hashes = lockfile.packages.items(.name_hash); @@ -10172,11 +10192,22 @@ pub const PackageManager = struct { }, }; + const existing_patchfile_hash = existing_patchfile_hash: { + var sfb = std.heap.stackFallback(1024, manager.allocator); + const name_and_version = std.fmt.allocPrint(sfb.get(), "{s}@{}", .{ name, actual_package.resolution.fmt(strbuf, .posix) }) catch unreachable; + defer sfb.get().free(name_and_version); + const name_and_version_hash = String.Builder.stringHash(name_and_version); + if (lockfile.patched_dependencies.get(name_and_version_hash)) |patched_dep| { + if (patched_dep.patchfileHash()) |hash| break :existing_patchfile_hash hash; + } + break :existing_patchfile_hash null; + }; + const cache_result = manager.computeCacheDirAndSubpath( name, &actual_package.resolution, &folder_path_buf, - null, + existing_patchfile_hash, ); const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; @@ -10218,7 +10249,24 @@ pub const PackageManager = struct { const pkg = manager.lockfile.packages.get(pkg_id); const pkg_name = pkg.name.slice(strbuf); - const cache_result = manager.computeCacheDirAndSubpath(pkg_name, &pkg.resolution, &folder_path_buf, null); + + const existing_patchfile_hash = existing_patchfile_hash: { + var sfb = std.heap.stackFallback(1024, manager.allocator); + const name_and_version = std.fmt.allocPrint(sfb.get(), "{s}@{}", .{ name, pkg.resolution.fmt(strbuf, .posix) }) catch unreachable; + defer sfb.get().free(name_and_version); + const name_and_version_hash = String.Builder.stringHash(name_and_version); + if (manager.lockfile.patched_dependencies.get(name_and_version_hash)) |patched_dep| { + if (patched_dep.patchfileHash()) |hash| break :existing_patchfile_hash hash; + } + break :existing_patchfile_hash null; + }; + + const cache_result = manager.computeCacheDirAndSubpath( + pkg_name, + &pkg.resolution, + &folder_path_buf, + existing_patchfile_hash, + ); const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; @@ -10233,6 +10281,10 @@ pub const PackageManager = struct { }, }; + // The package may be installed using the hard link method, + // meaning that changes to the folder will also change the package in the cache. + // + // So we will overwrite the folder by directly copying the package in cache into it manager.overwriteNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { Output.prettyError( "error: error overwriting folder in node_modules: {s}\n", @@ -10455,7 +10507,7 @@ pub const PackageManager = struct { }; // Attempt to open the existing node_modules folder - const root_node_modules = switch (bun.sys.openatOSPath(bun.FD.cwd(), bun.OSPathLiteral("node_modules"), std.os.O.DIRECTORY | std.os.O.RDONLY, 0o755)) { + var root_node_modules = switch (bun.sys.openatOSPath(bun.FD.cwd(), bun.OSPathLiteral("node_modules"), std.os.O.DIRECTORY | std.os.O.RDONLY, 0o755)) { .result => |fd| std.fs.Dir{ .fd = fd.cast() }, .err => |e| { Output.prettyError( @@ -10466,6 +10518,7 @@ pub const PackageManager = struct { Global.crash(); }, }; + defer root_node_modules.close(); var iterator = Lockfile.Tree.Iterator.init(lockfile); var resolution_buf: [1024]u8 = undefined; @@ -10557,8 +10610,8 @@ pub const PackageManager = struct { const name: []const u8, const version: ?[]const u8 = brk1: { if (std.mem.indexOfScalar(u8, argument[1..], '@')) |version_delimiter| { break :brk1 .{ - argument[0..version_delimiter], - argument[1..][version_delimiter + 1 ..], + argument[0 .. version_delimiter + 1], + argument[version_delimiter + 2 ..], }; } break :brk1 .{ @@ -10585,6 +10638,7 @@ pub const PackageManager = struct { name, }, .auto); const pkg = lockfile.packages.get(pkg_id); + const cache_result = manager.computeCacheDirAndSubpath( pkg.name.slice(lockfile.buffers.string_bytes.items), &pkg.resolution, @@ -10609,6 +10663,7 @@ pub const PackageManager = struct { const patchfile_contents = brk: { const new_folder = changes_dir; var buf2: bun.PathBuffer = undefined; + var buf3: bun.PathBuffer = undefined; const old_folder = old_folder: { const cache_dir_path = switch (bun.sys.getFdPath(bun.toFD(cache_dir.fd), &buf2)) { .result => |s| s, @@ -10635,6 +10690,12 @@ pub const PackageManager = struct { Output.flush(); Global.crash(); }); + + // If the package has nested a node_modules folder, we don't want this to + // appear in the patch file when we run git diff. + // + // There isn't an option to exclude it with `git diff --no-index`, so we + // will `rename()` it out and back again. const has_nested_node_modules = has_nested_node_modules: { var new_folder_handle = std.fs.cwd().openDir(new_folder, .{}) catch |e| { Output.prettyError( @@ -10655,8 +10716,52 @@ pub const PackageManager = struct { break :has_nested_node_modules true; }; + + const patch_tag_tmpname = bun.span(bun.fs.FileSystem.instance.tmpname(name, buf3[0..], bun.fastRandom()) catch |e| { + Output.prettyError( + "error: failed to make tempdir {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }); + + var bunpatchtagbuf: BuntagHashBuf = undefined; + // If the package was already patched then it might have a ".bun-tag-XXXXXXXX" + // we need to rename this out and back too. + const bun_patch_tag: ?[:0]const u8 = has_bun_patch_tag: { + const name_and_version_hash = String.Builder.stringHash(resolution_label); + const patch_tag = patch_tag: { + if (lockfile.patched_dependencies.get(name_and_version_hash)) |patchdep| { + if (patchdep.patchfileHash()) |hash| { + break :patch_tag buntaghashbuf_make(&bunpatchtagbuf, hash); + } + } + break :has_bun_patch_tag null; + }; + var new_folder_handle = std.fs.cwd().openDir(new_folder, .{}) catch |e| { + Output.prettyError( + "error: failed to open directory {s} {s}\n", + .{ new_folder, @errorName(e) }, + ); + Output.flush(); + Global.crash(); + }; + defer new_folder_handle.close(); + + if (bun.sys.renameatConcurrently( + bun.toFD(new_folder_handle.fd), + patch_tag, + bun.toFD(root_node_modules.fd), + patch_tag_tmpname, + ).asErr()) |e| { + Output.warn("failed renaming the bun patch tag, this may cause issues: {}", .{e}); + break :has_bun_patch_tag null; + } + break :has_bun_patch_tag patch_tag; + }; defer { - if (has_nested_node_modules) { + if (has_nested_node_modules or bun_patch_tag != null) { var new_folder_handle = std.fs.cwd().openDir(new_folder, .{}) catch |e| { Output.prettyError( "error: failed to open directory {s} {s}\n", @@ -10666,13 +10771,27 @@ pub const PackageManager = struct { Global.crash(); }; defer new_folder_handle.close(); - if (bun.sys.renameatConcurrently( - bun.toFD(root_node_modules.fd), - random_tempdir, - bun.toFD(new_folder_handle.fd), - "node_modules", - ).asErr()) |e| { - Output.warn("failed renaming nested node_modules folder, this may cause issues: {}", .{e}); + + if (has_nested_node_modules) { + if (bun.sys.renameatConcurrently( + bun.toFD(root_node_modules.fd), + random_tempdir, + bun.toFD(new_folder_handle.fd), + "node_modules", + ).asErr()) |e| { + Output.warn("failed renaming nested node_modules folder, this may cause issues: {}", .{e}); + } + } + + if (bun_patch_tag) |patch_tag| { + if (bun.sys.renameatConcurrently( + bun.toFD(root_node_modules.fd), + patch_tag_tmpname, + bun.toFD(new_folder_handle.fd), + patch_tag, + ).asErr()) |e| { + Output.warn("failed renaming the bun patch tag, this may cause issues: {}", .{e}); + } } } } @@ -13059,15 +13178,15 @@ pub const PackageManager = struct { _ = manager.getCacheDirectory(); _ = manager.getTemporaryDirectory(); } - manager.enqueueDependencyList(root.dependencies); { var iter = manager.lockfile.patched_dependencies.iterator(); - while (iter.next()) |entry| if (entry.value_ptr.patchfile_hash_is_null) manager.enqueuePatchTask(PatchTask.newCalcPatchHash(manager, entry.key_ptr.*, null)); + while (iter.next()) |entry| manager.enqueuePatchTaskPre(PatchTask.newCalcPatchHash(manager, entry.key_ptr.*, null)); } + manager.enqueueDependencyList(root.dependencies); } else { { var iter = manager.lockfile.patched_dependencies.iterator(); - while (iter.next()) |entry| if (entry.value_ptr.patchfile_hash_is_null) manager.enqueuePatchTask(PatchTask.newCalcPatchHash(manager, entry.key_ptr.*, null)); + while (iter.next()) |entry| manager.enqueuePatchTaskPre(PatchTask.newCalcPatchHash(manager, entry.key_ptr.*, null)); } // Anything that needs to be downloaded from an update needs to be scheduled here manager.drainDependencyList(); @@ -13087,7 +13206,7 @@ pub const PackageManager = struct { } const runAndWaitFn = struct { - pub fn runAndWaitFn(comptime check_peers: bool) *const fn (*PackageManager) anyerror!void { + pub fn runAndWaitFn(comptime check_peers: bool, comptime only_pre_patch: bool) *const fn (*PackageManager) anyerror!void { return struct { manager: *PackageManager, err: ?anyerror = null, @@ -13125,6 +13244,11 @@ pub const PackageManager = struct { } } + if (comptime only_pre_patch) { + const pending_patch = this.pending_pre_calc_hashes.load(.Monotonic); + return pending_patch == 0; + } + const pending_tasks = this.pendingTaskCount(); if (PackageManager.verbose_install and pending_tasks > 0) { @@ -13149,8 +13273,13 @@ pub const PackageManager = struct { } }.runAndWaitFn; - const waitForEverythingExceptPeers = runAndWaitFn(false); - const waitForPeers = runAndWaitFn(true); + const waitForCalcingPatchHashes = runAndWaitFn(false, true); + const waitForEverythingExceptPeers = runAndWaitFn(false, false); + const waitForPeers = runAndWaitFn(true, false); + + if (manager.lockfile.patched_dependencies.entries.len > 0) { + try waitForCalcingPatchHashes(manager); + } if (manager.pendingTaskCount() > 0) { try waitForEverythingExceptPeers(manager); diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 3686ef021aa4ec..de90c8f1921959 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -48,6 +48,7 @@ pub const PatchTask = struct { task: ThreadPool.Task = .{ .callback = runFromThreadPool, }, + pre: bool = false, next: ?*PatchTask = null, const debug = bun.Output.scoped(.InstallPatch, false); @@ -147,6 +148,9 @@ pub const PatchTask = struct { comptime log_level: PackageManager.Options.LogLevel, ) !void { debug("runFromThreadMainThread {s}", .{@tagName(this.callback)}); + defer { + if (this.pre) _ = manager.pending_pre_calc_hashes.fetchSub(1, .Monotonic); + } switch (this.callback) { .calc_hash => try this.runFromMainThreadCalcHash(manager, log_level), .apply => this.runFromMainThreadApply(manager), @@ -469,7 +473,6 @@ pub const PatchTask = struct { state: ?CalcPatchHash.EnqueueAfterState, ) *PatchTask { const patchdep = manager.lockfile.patched_dependencies.get(name_and_version_hash) orelse @panic("This is a bug"); - bun.debugAssert(patchdep.patchfile_hash_is_null); const patchfile_path = manager.allocator.dupeZ(u8, patchdep.path.slice(manager.lockfile.buffers.string_bytes.items)) catch bun.outOfMemory(); const pt = bun.new(PatchTask, .{ diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 2e0a86bcf70873..21ab1c2dad88be 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -4,7 +4,93 @@ import { afterAll, afterEach, beforeAll, beforeEach, expect, it, describe, test, import { join, sep } from "path"; describe("bun patch ", async () => { - test("should patch a package when it is already patched", async () => {}); + test("should patch a package when it is already patched", async () => { + const tempdir = tempDirWithFiles("lol", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + "is-odd": "3.0.1", + }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven(420))`, + }); + + await $`${bunExe()} i`.env(bunEnv).cwd(tempdir); + const { stderr } = await $`${bunExe()} patch is-odd@0.1.2`.env(bunEnv).cwd(tempdir).throws(false); + expect(stderr.toString()).not.toContain("error"); + + const firstChange = /* ts */ `/*! +* is-odd +* +* Copyright (c) 2015-2017, Jon Schlinkert. +* Released under the MIT License. +*/ + +'use strict'; + +var isNumber = require('is-number'); + +module.exports = function isOdd(i) { + if (!isNumber(i)) { + throw new TypeError('is-odd expects a number.'); + } + if (Number(i) !== Math.floor(i)) { + throw new RangeError('is-odd expects an integer.'); + } + console.log('hi') + return !!(~~i & 1); +};`; + + await $`echo ${firstChange} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(tempdir); + + const { stderr: stderr2 } = await $`${bunExe()} patch-commit node_modules/is-even/node_modules/is-odd` + .env(bunEnv) + .cwd(tempdir) + .throws(false); + expect(stderr2.toString()).not.toContain("error"); + + const { stderr: stderr3 } = await $`${bunExe()} patch is-odd@0.1.2`.env(bunEnv).cwd(tempdir).throws(false); + expect(stderr3.toString()).not.toContain("error"); + + const secondChange = /* ts */ `/*! +* is-odd +* +* Copyright (c) 2015-2017, Jon Schlinkert. +* Released under the MIT License. +*/ + +'use strict'; + +var isNumber = require('is-number'); + +module.exports = function isOdd(i) { + if (!isNumber(i)) { + throw new TypeError('is-odd expects a number.'); + } + if (Number(i) !== Math.floor(i)) { + throw new RangeError('is-odd expects an integer.'); + } + console.log('hi') + console.log('hello') + return !!(~~i & 1); +};`; + + await $`echo ${secondChange} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(tempdir); + const { stderr: stderr4 } = await $`${bunExe()} patch-commit node_modules/is-even/node_modules/is-odd` + .env(bunEnv) + .cwd(tempdir) + .throws(false); + expect(stderr4.toString()).not.toContain("error"); + + await $`cat patches/is-odd@0.1.2.patch`.env(bunEnv).cwd(tempdir); + + await $`${bunExe()} i`.env(bunEnv).cwd(tempdir).throws(false); + const { stdout } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir).throws(false); + expect(stdout.toString()).toContain("hi\nhello\n"); + }); test("bad patch arg", async () => { const tempdir = tempDirWithFiles("lol", { From d58726e21da43590879cc4bdd27e47ec2c558a5f Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Tue, 11 Jun 2024 20:22:16 -0700 Subject: [PATCH 09/51] use correct path for .verify on patched packages --- src/install/install.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 314de0f3d7f6b7..b3b53c79c149a8 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -1052,10 +1052,15 @@ pub fn NewPackageInstall(comptime kind: PkgInstallKind) type { bunhashtag, }, .posix); + var destination_dir = this.node_modules.openDir(root_node_modules_dir) catch return false; + defer { + if (std.fs.cwd().fd != destination_dir.fd) destination_dir.close(); + } + if (comptime bun.Environment.isPosix) { - _ = bun.sys.fstatat(bun.toFD(root_node_modules_dir.fd), patch_tag_path).unwrap() catch return false; + _ = bun.sys.fstatat(bun.toFD(destination_dir.fd), patch_tag_path).unwrap() catch return false; } else { - switch (bun.sys.openat(bun.toFD(root_node_modules_dir.fd), patch_tag_path, std.os.O.RDONLY, 0)) { + switch (bun.sys.openat(bun.toFD(destination_dir.fd), patch_tag_path, std.os.O.RDONLY, 0)) { .err => return false, .result => |fd| _ = bun.sys.close(fd), } From 198ec6c5b9f83c9754c06098d5c5cf09736d3938 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Tue, 11 Jun 2024 20:44:32 -0700 Subject: [PATCH 10/51] better error handing there --- src/install/patch_install.zig | 128 ++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 52 deletions(-) diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index de90c8f1921959..46c6503629ae4a 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -10,6 +10,7 @@ const strings = bun.strings; const MutableString = bun.MutableString; const logger = bun.logger; +const Loc = logger.Loc; const PackageManager = bun.PackageManager; pub const PackageID = bun.install.PackageID; @@ -53,11 +54,6 @@ pub const PatchTask = struct { const debug = bun.Output.scoped(.InstallPatch, false); - fn errDupePath(e: bun.sys.Error) bun.sys.Error { - if (e.path.len > 0) return e.withPath(bun.default_allocator.dupe(u8, e.path) catch bun.outOfMemory()); - return e; - } - const Maybe = bun.sys.Maybe; const CalcPatchHash = struct { @@ -66,7 +62,9 @@ pub const PatchTask = struct { state: ?EnqueueAfterState = null, - result: ?Maybe(u64) = null, + result: ?u64 = null, + + logger: logger.Log, const EnqueueAfterState = struct { pkg_id: PackageID, @@ -106,15 +104,12 @@ pub const PatchTask = struct { this.manager.allocator.free(this.callback.apply.cache_dir_subpath); this.manager.allocator.free(this.callback.apply.pkgname); if (this.callback.apply.install_context) |ictx| ictx.path.deinit(); + this.callback.apply.logger.deinit(); }, .calc_hash => { // TODO: how to deinit `this.callback.calc_hash.network_task` if (this.callback.calc_hash.state) |state| this.manager.allocator.free(state.url); - if (this.callback.calc_hash.result) |r| { - if (r.asErr()) |e| { - if (e.path.len > 0) bun.default_allocator.free(e.path); - } - } + this.callback.calc_hash.logger.deinit(); this.manager.allocator.free(this.callback.calc_hash.patchfile_path); }, } @@ -174,42 +169,23 @@ pub const PatchTask = struct { // TODO only works for npm package // need to switch on version.tag and handle each case appropriately const calc_hash = &this.callback.calc_hash; - const hash = switch (calc_hash.result orelse @panic("Calc hash didn't run, this is a bug in Bun.")) { - .result => |h| h, - .err => |e| { - if (e.getErrno() == bun.C.E.NOENT) { - const fmt = "\n\nerror: could not find patch file {s}\n\nPlease make sure it exists.\n\nTo create a new patch file run:\n\n bun patch {s}\n"; - const args = .{ - this.callback.calc_hash.patchfile_path, - manager.lockfile.patched_dependencies.get(calc_hash.name_and_version_hash).?.path.slice(manager.lockfile.buffers.string_bytes.items), - }; - if (comptime log_level.showProgress()) { - Output.prettyWithPrinterFn(fmt, args, Progress.log, &manager.progress); - } else { - Output.prettyErrorln( - fmt, - args, - ); - Output.flush(); - } - Global.crash(); - } - - const fmt = "\n\nerror: {s}{s} while calculating hash for patchfile: {s}\n"; - const args = .{ @tagName(e.getErrno()), e.path, this.callback.calc_hash.patchfile_path }; - if (comptime log_level.showProgress()) { - Output.prettyWithPrinterFn(fmt, args, Progress.log, &manager.progress); - } else { - Output.prettyErrorln( - fmt, - args, - ); - Output.flush(); - } - Global.crash(); - - return; - }, + const hash = calc_hash.result orelse { + const fmt = "\n\nErrors occured while calculating hash for {s}:\n\n"; + const args = .{this.callback.calc_hash.patchfile_path}; + if (comptime log_level.showProgress()) { + Output.prettyWithPrinterFn(fmt, args, Progress.log, &manager.progress); + } else { + Output.prettyErrorln( + fmt, + args, + ); + } + if (calc_hash.logger.errors > 0) { + Output.prettyErrorln("\n\n", .{}); + calc_hash.logger.printForLogLevel(Output.writer()) catch {}; + } + Output.flush(); + Global.crash(); }; var gop = manager.lockfile.patched_dependencies.getOrPut(manager.allocator, calc_hash.name_and_version_hash) catch bun.outOfMemory(); @@ -412,8 +388,9 @@ pub const PatchTask = struct { ).asErr()) |e| return try log.addErrorFmtNoLoc(this.manager.allocator, "{}", .{e}); } - pub fn calcHash(this: *PatchTask) Maybe(u64) { + pub fn calcHash(this: *PatchTask) ?u64 { bun.assert(this.callback == .calc_hash); + var log = &this.callback.calc_hash.logger; const dir = this.project_dir; const patchfile_path = this.callback.calc_hash.patchfile_path; @@ -425,13 +402,50 @@ pub const PatchTask = struct { }, .auto); const stat: bun.Stat = switch (bun.sys.stat(absolute_patchfile_path)) { - .err => |e| return .{ .err = errDupePath(e) }, + .err => |e| { + if (e.getErrno() == bun.C.E.NOENT) { + const fmt = "\n\nerror: could not find patch file {s}\n\nPlease make sure it exists.\n\nTo create a new patch file run:\n\n bun patch {s}\n"; + const args = .{ + this.callback.calc_hash.patchfile_path, + this.manager.lockfile.patched_dependencies.get(this.callback.calc_hash.name_and_version_hash).?.path.slice(this.manager.lockfile.buffers.string_bytes.items), + }; + log.addErrorFmt(null, Loc.Empty, this.manager.allocator, fmt, args) catch bun.outOfMemory(); + return null; + } + log.addWarningFmt( + null, + Loc.Empty, + this.manager.allocator, + "patchfile {s} is empty", + .{absolute_patchfile_path}, + ) catch bun.outOfMemory(); + return null; + }, .result => |s| s, }; const size: u64 = @intCast(stat.size); + if (size == 0) { + log.addErrorFmt( + null, + Loc.Empty, + this.manager.allocator, + "patchfile {s} is empty", + .{absolute_patchfile_path}, + ) catch bun.outOfMemory(); + return null; + } const fd = switch (bun.sys.open(absolute_patchfile_path, std.os.O.RDONLY, 0)) { - .err => |e| return .{ .err = errDupePath(e) }, + .err => |e| { + log.addErrorFmt( + null, + Loc.Empty, + this.manager.allocator, + "failed to open patch file: {}", + .{e}, + ) catch bun.outOfMemory(); + return null; + }, .result => |fd| fd, }; defer _ = bun.sys.close(fd); @@ -448,14 +462,23 @@ pub const PatchTask = struct { while (i < STACK_SIZE and i < size) { switch (bun.sys.read(fd, stack[i..])) { .result => |w| i += w, - .err => |e| return .{ .err = errDupePath(e) }, + .err => |e| { + log.addErrorFmt( + null, + Loc.Empty, + this.manager.allocator, + "failed to read from patch file: {} ({s})", + .{ e, absolute_patchfile_path }, + ) catch bun.outOfMemory(); + return null; + }, } } read += i; hasher.update(stack[0..i]); } - return .{ .result = hasher.final() }; + return hasher.final(); } pub fn notify(this: *PatchTask) void { @@ -481,6 +504,7 @@ pub const PatchTask = struct { .state = state, .patchfile_path = patchfile_path, .name_and_version_hash = name_and_version_hash, + .logger = logger.Log.init(manager.allocator), }, }, .manager = manager, From 4c034fcb12bf8e162da85f19caf54c0eb81afe71 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:25:16 -0700 Subject: [PATCH 11/51] patch --commit --- src/install/install.zig | 122 ++++++++++++++++++----------- src/install/patch_install.zig | 4 +- src/patch.zig | 72 +++++++++++------ test/cli/install/bun-patch.test.ts | 10 +-- 4 files changed, 130 insertions(+), 78 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index b3b53c79c149a8..2becd53808c35a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -6723,11 +6723,12 @@ pub const PackageManager = struct { .workspaces = true, }, patch_features: union(enum) { + NOTHING: struct {}, patch: struct {}, commit: struct { patches_dir: string, }, - } = .{ .patch = .{} }, + } = .{ .NOTHING = .{} }, // The idea here is: // 1. package has a platform-specific binary to install // 2. To prevent downloading & installing incompatible versions, they stick the "real" one in optionalDependencies @@ -7189,15 +7190,28 @@ pub const PackageManager = struct { this.update.development = cli.development; if (!this.update.development) this.update.optional = cli.optional; - if (subcommand == .patch) { - // TODO args - } else if (subcommand == .@"patch-commit") { - this.patch_features = .{ - .commit = .{ - .patches_dir = cli.@"patch-commit".patches_dir, - }, - }; + switch (cli.patch) { + .NOTHING => {}, + .patch => { + this.patch_features = .{ .patch = .{} }; + }, + .commit => { + this.patch_features = .{ + .commit = .{ + .patches_dir = cli.patch.commit.patches_dir, + }, + }; + }, } + // if (subcommand == .patch) { + // // TODO args + // } else if (subcommand == .@"patch-commit") { + // this.patch_features = .{ + // .commit = .{ + // .patches_dir = cli.@"patch-commit".patches_dir, + // }, + // }; + // } } else { this.log_level = if (default_disable_progress_bar) LogLevel.default_no_progress else LogLevel.default; PackageManager.verbose_install = false; @@ -9016,6 +9030,8 @@ pub const PackageManager = struct { const patch_params = install_params_ ++ [_]ParamType{ clap.parseParam(" ... \"name\" of the package to patch") catch unreachable, + clap.parseParam("--commit Install a package containing modifications in `dir`") catch unreachable, + clap.parseParam("--patches-dir The directory to put the patch file in (only if --commit is used)") catch unreachable, }; const patch_commit_params = install_params_ ++ [_]ParamType{ @@ -9063,16 +9079,14 @@ pub const PackageManager = struct { concurrent_scripts: ?usize = null, - patch: PatchOpts = .{}, - @"patch-commit": PatchCommitOpts = .{}, + patch: PatchOpts = .{ .NOTHING = .{} }, - const PatchOpts = struct { - edit_dir: ?[]const u8 = null, - ignore_existing: bool = false, - }; - - const PatchCommitOpts = struct { - patches_dir: []const u8 = "patches", + const PatchOpts = union(enum) { + NOTHING: struct {}, + patch: struct {}, + commit: struct { + patches_dir: []const u8 = "patches", + }, }; const Omit = struct { @@ -9340,16 +9354,30 @@ pub const PackageManager = struct { cli.no_save = args.flag("--no-save"); } - if (comptime subcommand == .patch) { - cli.patch = .{}; + if (subcommand == .patch) { + const patch_commit = args.flag("--commit"); + if (patch_commit) { + cli.patch = .{ + .commit = .{ + .patches_dir = args.option("--patches-dir") orelse "patches", + }, + }; + } else { + cli.patch = .{ + .patch = .{}, + }; + } } - - if (comptime subcommand == .@"patch-commit") { - cli.@"patch-commit" = .{ - .patches_dir = args.option("--patches-dir") orelse "patches", + if (subcommand == .@"patch-commit") { + cli.patch = .{ + .commit = .{ + .patches_dir = args.option("--patches-dir") orelse "patches", + }, }; } + if (comptime subcommand == .@"patch-commit") {} + if (args.option("--config")) |opt| { cli.config = opt; } @@ -9600,7 +9628,7 @@ pub const PackageManager = struct { Output.prettyErrorln("No package.json, so nothing to remove\n", .{}); Global.crash(); }, - .patch => { + .patch, .@"patch-commit" => { Output.prettyErrorln("No package.json, so nothing to patch\n", .{}); Global.crash(); }, @@ -9623,7 +9651,7 @@ pub const PackageManager = struct { inline else => |log_level| try manager.updatePackageJSONAndInstallWithManager(ctx, log_level), } - if (comptime subcommand == .patch) { + if (manager.options.patch_features == .patch) { try manager.preparePatch(); } @@ -9800,24 +9828,7 @@ pub const PackageManager = struct { } } }, - .@"patch-commit" => { - // _ = manager.lockfile.loadFromDisk( - // manager, - // manager.allocator, - // manager.log, - // manager.options.lockfile_path, - // true, - // ); - var pathbuf: bun.PathBuffer = undefined; - if (try manager.doPatchCommit(&pathbuf, log_level)) |stuff| { - try PackageJSONEditor.editPatchedDependencies( - manager, - ¤t_package_json.root, - stuff.patch_key, - stuff.patchfile_path, - ); - } - }, + .link, .add, .update => { // `bun update ` is basically the same as `bun add `, except // update will not exceed the current dependency range if it exists @@ -9844,7 +9855,26 @@ pub const PackageManager = struct { ); } }, - else => {}, + else => { + if (manager.options.patch_features == .commit) { + // _ = manager.lockfile.loadFromDisk( + // manager, + // manager.allocator, + // manager.log, + // manager.options.lockfile_path, + // true, + // ); + var pathbuf: bun.PathBuffer = undefined; + if (try manager.doPatchCommit(&pathbuf, log_level)) |stuff| { + try PackageJSONEditor.editPatchedDependencies( + manager, + ¤t_package_json.root, + stuff.patch_key, + stuff.patchfile_path, + ); + } + } + }, } manager.to_update = subcommand == .update; @@ -10300,7 +10330,7 @@ pub const PackageManager = struct { }; Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ pkg_name, module_folder }); - Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch-commit '{s}'\n", .{module_folder}); + Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch --commit '{s}'\n", .{module_folder}); return; } diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 46c6503629ae4a..7fc4617b567b9c 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -416,7 +416,7 @@ pub const PatchTask = struct { null, Loc.Empty, this.manager.allocator, - "patchfile {s} is empty", + "patchfile {s} is empty, please restore or delete it.", .{absolute_patchfile_path}, ) catch bun.outOfMemory(); return null; @@ -429,7 +429,7 @@ pub const PatchTask = struct { null, Loc.Empty, this.manager.allocator, - "patchfile {s} is empty", + "patchfile {s} is empty, plese restore or delete it.", .{absolute_patchfile_path}, ) catch bun.outOfMemory(); return null; diff --git a/src/patch.zig b/src/patch.zig index eb10cd7b333fd0..f0c187dafd8acc 100644 --- a/src/patch.zig +++ b/src/patch.zig @@ -1329,8 +1329,10 @@ pub fn gitDiff( /// - b/src/index.js /// /// The operations look roughy like the following sequence of substitutions and regexes: -/// .replace(new RegExp(`(a|b)(${escapeStringRegexp(`/${removeTrailingAndLeadingSlash(aFolder)}/`)})`, "g"), "$1/") -/// .replace(new RegExp(`(a|b)${escapeStringRegexp(`/${removeTrailingAndLeadingSlash(bFolder)}/`)}`, "g"), "$1/") +/// .replace(new RegExp(`(a|b)(${escapeStringRegexp(`/${removeTrailingAndLeadingSlash(aFolder)}/`)})`, "g"), "$1/") +/// .replace(new RegExp(`(a|b)${escapeStringRegexp(`/${removeTrailingAndLeadingSlash(bFolder)}/`)}`, "g"), "$1/") +/// .replace(new RegExp(escapeStringRegexp(`${aFolder}/`), "g"), "") +/// .replace(new RegExp(escapeStringRegexp(`${bFolder}/`), "g"), ""); fn gitDiffPostprocess(stdout: *std.ArrayList(u8), old_folder: []const u8, new_folder: []const u8) !void { const old_folder_trimmed = std.mem.trim(u8, old_folder, "/"); const new_folder_trimmed = std.mem.trim(u8, new_folder, "/"); @@ -1355,39 +1357,59 @@ fn gitDiffPostprocess(stdout: *std.ArrayList(u8), old_folder: []const u8, new_fo // const @"$old_folder/" = @"a/$old_folder/"[2..]; // const @"$new_folder/" = @"b/$new_folder/"[2..]; + // these vars are here to disambguate `a/$OLD_FOLDER` when $OLD_FOLDER itself contains "a/" + // basically if $OLD_FOLDER contains "a/" then the code will replace it + // so we need to not run that code path + var saw_a_folder: ?usize = null; + var saw_b_folder: ?usize = null; + var line_idx: u32 = 0; + var line_iter = std.mem.splitScalar(u8, stdout.items, '\n'); while (line_iter.next()) |line| { - if (shouldSkipLine(line)) continue; - if (std.mem.indexOf(u8, line, @"a/$old_folder/")) |idx| { - const @"$old_folder/ start" = idx + 2; - const line_start = line_iter.index.? - 1 - line.len; - line_iter.index.? -= 1 + line.len; - try stdout.replaceRange(line_start + @"$old_folder/ start", old_folder_trimmed.len + 1, ""); - continue; - } - if (std.mem.indexOf(u8, line, @"b/$new_folder/")) |idx| { - const @"$new_folder/ start" = idx + 2; - const line_start = line_iter.index.? - 1 - line.len; - try stdout.replaceRange(line_start + @"$new_folder/ start", new_folder_trimmed.len + 1, ""); - line_iter.index.? -= new_folder_trimmed.len + 1; - continue; - } - if (std.mem.indexOf(u8, line, old_folder)) |idx| { - if (idx + old_folder.len < line.len and line[idx + old_folder.len] == '/') { + if (!shouldSkipLine(line)) { + if (std.mem.indexOf(u8, line, @"a/$old_folder/")) |idx| { + const @"$old_folder/ start" = idx + 2; const line_start = line_iter.index.? - 1 - line.len; line_iter.index.? -= 1 + line.len; - try stdout.replaceRange(line_start + idx, old_folder.len + 1, ""); + try stdout.replaceRange(line_start + @"$old_folder/ start", old_folder_trimmed.len + 1, ""); + saw_a_folder = line_idx; continue; } - } - if (std.mem.indexOf(u8, line, new_folder)) |idx| { - if (idx + new_folder.len < line.len and line[idx + new_folder.len] == '/') { + if (std.mem.indexOf(u8, line, @"b/$new_folder/")) |idx| { + const @"$new_folder/ start" = idx + 2; const line_start = line_iter.index.? - 1 - line.len; - line_iter.index.? -= 1 + line.len; - try stdout.replaceRange(line_start + idx, new_folder.len + 1, ""); + try stdout.replaceRange(line_start + @"$new_folder/ start", new_folder_trimmed.len + 1, ""); + line_iter.index.? -= new_folder_trimmed.len + 1; + saw_b_folder = line_idx; continue; } + if (saw_a_folder == null or saw_a_folder.? != line_idx) { + if (std.mem.indexOf(u8, line, old_folder)) |idx| { + if (idx + old_folder.len < line.len and line[idx + old_folder.len] == '/') { + const line_start = line_iter.index.? - 1 - line.len; + line_iter.index.? -= 1 + line.len; + try stdout.replaceRange(line_start + idx, old_folder.len + 1, ""); + saw_a_folder = line_idx; + continue; + } + } + } + if (saw_b_folder == null or saw_b_folder.? != line_idx) { + if (std.mem.indexOf(u8, line, new_folder)) |idx| { + if (idx + new_folder.len < line.len and line[idx + new_folder.len] == '/') { + const line_start = line_iter.index.? - 1 - line.len; + line_iter.index.? -= 1 + line.len; + try stdout.replaceRange(line_start + idx, new_folder.len + 1, ""); + saw_b_folder = line_idx; + continue; + } + } + } } + + line_idx += 1; + saw_a_folder = null; + saw_b_folder = null; } } diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 21ab1c2dad88be..77c9fc9ce5fed4 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -46,7 +46,7 @@ module.exports = function isOdd(i) { await $`echo ${firstChange} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(tempdir); - const { stderr: stderr2 } = await $`${bunExe()} patch-commit node_modules/is-even/node_modules/is-odd` + const { stderr: stderr2 } = await $`${bunExe()} patch --commit node_modules/is-even/node_modules/is-odd` .env(bunEnv) .cwd(tempdir) .throws(false); @@ -79,7 +79,7 @@ module.exports = function isOdd(i) { };`; await $`echo ${secondChange} > node_modules/is-even/node_modules/is-odd/index.js`.env(bunEnv).cwd(tempdir); - const { stderr: stderr4 } = await $`${bunExe()} patch-commit node_modules/is-even/node_modules/is-odd` + const { stderr: stderr4 } = await $`${bunExe()} patch --commit node_modules/is-even/node_modules/is-odd` .env(bunEnv) .cwd(tempdir) .throws(false); @@ -128,7 +128,7 @@ module.exports = function isOdd(i) { const { stderr } = await $`${bunExe()} patch is-even`.env(bunEnv).cwd(tempdir); expect(stderr.toString()).not.toContain("error"); - const { stderr: stderr2 } = await $`${bunExe()} patch-commit lskfjdslkfjsldkfjlsdkfj` + const { stderr: stderr2 } = await $`${bunExe()} patch --commit lskfjdslkfjsldkfjlsdkfj` .env(bunEnv) .cwd(tempdir) .throws(false); @@ -180,7 +180,7 @@ module.exports = function isOdd(i) { Once you're done with your changes, run: - bun patch-commit '${expected.patchPath}'`, + bun patch --commit '${expected.patchPath}'`, ); } @@ -188,7 +188,7 @@ Once you're done with your changes, run: const newCode = patchedCode; await $`echo ${newCode} > ${expected.patchPath}/index.js`.env(bunEnv).cwd(filedir); - const { stderr, stdout } = await $`${bunExe()} patch-commit ${expected.patchPath}`.env(bunEnv).cwd(filedir); + const { stderr, stdout } = await $`${bunExe()} patch --commit ${expected.patchPath}`.env(bunEnv).cwd(filedir); } const output = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(filedir).text(); From 49f515669604e5c51ba6fc901d0e143afb82f35a Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:10:28 -0700 Subject: [PATCH 12/51] prevent stucking on Resolve... --- src/install/install.zig | 58 +++++++++++----------- src/install/patch_install.zig | 2 +- test/cli/install/bun-install-patch.test.ts | 47 ++++++++++++++++++ 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 2becd53808c35a..bc045ae8d318f0 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -5998,34 +5998,36 @@ pub const PackageManager = struct { defer ptask.deinit(); try ptask.runFromMainThread(manager, log_level); if (ptask.callback == .apply) { - if (comptime @TypeOf(callbacks.onExtract) != void) { - if (ptask.callback.apply.task_id) |task_id| { - _ = task_id; // autofix - - // const name = manager.lockfile.packages.items(.name)[ptask.callback.apply.pkg_id].slice(manager.lockfile.buffers.string_bytes.items); - // if (!callbacks.onPatch(extract_ctx, name, task_id, log_level)) { - // if (comptime Environment.allow_assert) { - // Output.panic("Ran callback to install enqueued packages, but there was no task associated with it.", .{}); - // } - // } - } else if (ExtractCompletionContext == *PackageInstaller) { - if (ptask.callback.apply.install_context) |*ctx| { - var installer: *PackageInstaller = extract_ctx; - const path = ctx.path; - ctx.path = std.ArrayList(u8).init(bun.default_allocator); - installer.node_modules.path = path; - installer.current_tree_id = ctx.tree_id; - const pkg_id = ptask.callback.apply.pkg_id; - - installer.installPackageWithNameAndResolution( - ctx.dependency_id, - pkg_id, - log_level, - ptask.callback.apply.pkgname, - ptask.callback.apply.resolution, - false, - false, - ); + if (ptask.callback.apply.logger.errors == 0) { + if (comptime @TypeOf(callbacks.onExtract) != void) { + if (ptask.callback.apply.task_id) |task_id| { + _ = task_id; // autofix + + // const name = manager.lockfile.packages.items(.name)[ptask.callback.apply.pkg_id].slice(manager.lockfile.buffers.string_bytes.items); + // if (!callbacks.onPatch(extract_ctx, name, task_id, log_level)) { + // if (comptime Environment.allow_assert) { + // Output.panic("Ran callback to install enqueued packages, but there was no task associated with it.", .{}); + // } + // } + } else if (ExtractCompletionContext == *PackageInstaller) { + if (ptask.callback.apply.install_context) |*ctx| { + var installer: *PackageInstaller = extract_ctx; + const path = ctx.path; + ctx.path = std.ArrayList(u8).init(bun.default_allocator); + installer.node_modules.path = path; + installer.current_tree_id = ctx.tree_id; + const pkg_id = ptask.callback.apply.pkg_id; + + installer.installPackageWithNameAndResolution( + ctx.dependency_id, + pkg_id, + log_level, + ptask.callback.apply.pkgname, + ptask.callback.apply.resolution, + false, + false, + ); + } } } } diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 7fc4617b567b9c..b8606cf6bbe0fe 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -251,7 +251,7 @@ pub const PatchTask = struct { // 5. Add bun tag for patch hash // 6. rename() newly patched pkg to cache pub fn apply(this: *PatchTask) !void { - var log = this.callback.apply.logger; + var log = &this.callback.apply.logger; debug("apply patch task", .{}); bun.assert(this.callback == .apply); diff --git a/test/cli/install/bun-install-patch.test.ts b/test/cli/install/bun-install-patch.test.ts index 6bf623a2668936..c8375d9647e1e4 100644 --- a/test/cli/install/bun-install-patch.test.ts +++ b/test/cli/install/bun-install-patch.test.ts @@ -482,4 +482,51 @@ index aa7c7012cda790676032d1b01d78c0b69ec06360..6048e7cb462b3f9f6ac4dc21aacf9a09 expect(stderr.toString()).toBe(""); expect(stdout.toString()).toContain("PATCHED!\n"); }); + + it("shouldn't infinite loop on failure to apply patch", async () => { + const badPatch = /* patch */ `diff --git a/index.js b/node_modules/is-even/index.js +index 832d92223a9ec491364ee10dcbe3ad495446ab80..7e079a817825de4b8c3d01898490dc7e960172bb 100644 +--- a/index.js ++++ b/node_modules/is-even/index.js +@@ -10,5 +10,6 @@ + var isOdd = require('is-odd'); + + module.exports = function isEven(i) { ++ console.log('hi') + return !isOdd(i); + }; +`; + + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + "is-even": "1.0.0", + }, + }), + patches: { + "is-even@1.0.0.patch": badPatch, + }, + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); + + await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + + const pkgjsonWithPatch = { + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "patchedDependencies": { + "is-even@1.0.0": "patches/is-even@1.0.0.patch", + }, + "dependencies": { + "is-even": "1.0.0", + }, + }; + + await $`echo ${JSON.stringify(pkgjsonWithPatch)} > package.json`.cwd(filedir).env(bunEnv); + await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + }); }); From e19970794fa06851a43c37981bcd11cb7e2722ff Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:10:59 -0700 Subject: [PATCH 13/51] docs --- docs/cli/patch-commit.md | 4 +++- docs/cli/patch.md | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/cli/patch-commit.md b/docs/cli/patch-commit.md index 159022d08bbd31..e20dbaeb4f93f0 100644 --- a/docs/cli/patch-commit.md +++ b/docs/cli/patch-commit.md @@ -1,4 +1,6 @@ -After having prepared a package for patching with [`bun patch`](/docs/cli/patch), you can install with `bun patch-commit `. +An alias for `bun patch --commit` to maintain compatibility with pnpm. + +You must prepare the package for patching with [`bun patch `](/docs/cli/patch) first. ### `--patches-dir` diff --git a/docs/cli/patch.md b/docs/cli/patch.md index 922446543d898f..a3873197112f01 100644 --- a/docs/cli/patch.md +++ b/docs/cli/patch.md @@ -1,10 +1,42 @@ -If you need to modify the contents of a package, call `bun patch ` with the package's name (and optionally a version), -for example: +Bun lets you easily make quick fixes to packages and have those changes work consistently across multiple installs and machines, without having to go through the work of forking and publishing a new version of the package. + +To get started, use `bun patch ` to prepare it for patching: ```bash +# you can supply the package name $ bun patch react + +# ...and a precise version in case multiple versions are installed +$ bun patch react@17.0.2 + +# or the path to the package +$ bun patch node_modules/react ``` -This will copy the package to a temporary directory, where you can make changes to the package's contents. +The output of this command will give you the path to the package in `node_modules/` where you can make your changes to the package. + +This allows you to test your changes before committing them. + +{% callout %} +**Note** — Don't forget to call `bun patch `! This ensures the package folder in `node_modules/` contains a fresh copy of the package with no symlinks/hardlinks to Bun's cache. + +If you forget to do this, you might end up editing the package globally in the cache! +{% /callout %} + +Once you're happy with your changes, run `bun patch --commit `. -Once you're done making changes, run `bun patch-commit ` to have Bun install the patched package. +Bun will generate a patch file in `patches/`, update your `package.json` and lockfile, and Bun will start using the patched package: + +```bash +# you can supply the path to the patched package +$ bun patch --commit node_modules/react + +# ... or the package name and optionally the version +$ bun patch --commit react@17.0.2 + +# choose the directory to store the patch files +$ bun patch --commit react --patches-dir=mypatches + +# `patch-commit` is available for compatibility with pnpm +$ bun patch-commit react +``` From 23ebf4d6e5710c9f53635c1a244e5a5b790199a9 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 16:43:25 -0700 Subject: [PATCH 14/51] Fix help spacing --- src/cli.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli.zig b/src/cli.zig index af883346a78210..a5591a1558f54f 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -1005,8 +1005,8 @@ pub const HelpCommand = struct { \\ update {s:<16} Update outdated dependencies \\ link [\] Register or link a local npm package \\ unlink Unregister a local npm package - \\ patch \ Prepare a package for patching - \\ patch-commit \ Install a modified package + \\ patch \ Prepare a package for patching + \\ patch-commit \ Install a modified package \\ pm \ Additional package management utilities \\ \\ build ./a.ts ./b.jsx Bundle TypeScript & JavaScript into a single file From e5ba08ca40c5d83407f7ef2976ba26c0f36d1f65 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:39:02 -0700 Subject: [PATCH 15/51] windows stuff --- src/install/install.zig | 335 +++++++++++++++++++++++++++------------- src/sys.zig | 5 +- 2 files changed, 232 insertions(+), 108 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index bc045ae8d318f0..37ea1095583836 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10322,14 +10322,42 @@ pub const PackageManager = struct { // meaning that changes to the folder will also change the package in the cache. // // So we will overwrite the folder by directly copying the package in cache into it - manager.overwriteNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { - Output.prettyError( - "error: error overwriting folder in node_modules: {s}\n", - .{@errorName(e)}, - ); - Output.flush(); - Global.crash(); - }; + if (comptime bun.Environment.isWindows) { + const modules_parent_dir = bun.path.dirname(module_folder, .auto); + var parent_dir = std.fs.cwd().openDir(modules_parent_dir, .{ .iterate = true }) catch |e| { + Output.prettyError( + "error: error opening folder in node_modules: {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }; + defer parent_dir.close(); + + manager.overwriteNodeModulesFolderWindows( + cache_dir, + cache_dir_subpath, + module_folder, + parent_dir, + bun.path.basename(module_folder), + ) catch |e| { + Output.prettyError( + "error: error overwriting folder in node_modules: {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }; + } else { + manager.overwriteNodeModulesFolderPosix(cache_dir, cache_dir_subpath, module_folder) catch |e| { + Output.prettyError( + "error: error overwriting folder in node_modules: {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }; + } Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ pkg_name, module_folder }); Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch --commit '{s}'\n", .{module_folder}); @@ -10337,7 +10365,7 @@ pub const PackageManager = struct { return; } - fn overwriteNodeModulesFolder( + fn overwriteNodeModulesFolderPosix( manager: *PackageManager, cache_dir: std.fs.Dir, cache_dir_subpath: []const u8, @@ -10346,117 +10374,160 @@ pub const PackageManager = struct { var node_modules_folder = try std.fs.cwd().openDir(node_modules_folder_path, .{ .iterate = true }); defer node_modules_folder.close(); - const IGNORED_PATHS: []const bun.OSPathSlice = &.{ - "node_modules", - ".git", - "CMakeFiles", + const IGNORED_PATHS: []const bun.OSPathSlice = &[_][]const bun.OSPathChar{ + bun.OSPathLiteral("node_modules"), + bun.OSPathLiteral(".git"), + bun.OSPathLiteral("CMakeFiles"), }; const FileCopier = struct { pub fn copy( destination_dir_: std.fs.Dir, walker: *Walker, - to_copy_into1: if (Environment.isWindows) []u16 else void, - head1: if (Environment.isWindows) []u16 else void, - to_copy_into2: if (Environment.isWindows) []u16 else void, - head2: if (Environment.isWindows) []u16 else void, ) !u32 { var real_file_count: u32 = 0; var copy_file_state: bun.CopyFileState = .{}; while (try walker.next()) |entry| { - if (comptime Environment.isWindows) { - switch (entry.kind) { - .directory, .file => {}, - else => continue, - } + if (entry.kind != .file) continue; + real_file_count += 1; + const openFile = std.fs.Dir.openFile; + const createFile = std.fs.Dir.createFile; + + var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); + defer in_file.close(); + + if (bun.sys.unlinkat( + bun.toFD(destination_dir_.fd), + entry.basename.ptr[0..entry.basename.len :0], + ).asErr()) |e| { + Output.prettyError("error: copying file {}", .{e}); + Global.crash(); + } - if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { - return error.NameTooLong; - } + const mode = try in_file.mode(); + var outfile = try createFile(destination_dir_, entry.basename, .{ .mode = mode }); + defer outfile.close(); - @memcpy(to_copy_into1[0..entry.path.len], entry.path); - head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; - const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; + debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); + // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { + // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { + // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; + // } + // break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { + // if (do_progress) { + // progress_.root.end(); + // progress_.refresh(); + // } - @memcpy(to_copy_into2[0..entry.path.len], entry.path); - head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; - const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; + // Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + // Global.crash(); + // }; + // }; + // defer outfile.close(); - switch (entry.kind) { - .directory => { - if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { - bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; - } - }, - .file => { - if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) == 0) { - if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { - bun.MakePath.makePath(u16, destination_dir_, entry_dirname) catch {}; - if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) != 0) { - continue; - } - } + if (comptime Environment.isPosix) { + const stat = in_file.stat() catch continue; + _ = C.fchmod(outfile.handle, @intCast(stat.mode)); + } - if (bun.windows.Win32Error.get().toSystemErrno()) |err| { - Output.prettyError("{s}: copying file {}", .{ @tagName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - } else { - Output.prettyError("error copying file {}", .{bun.fmt.fmtOSPath(entry.path, .{})}); - } + bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; + } - Global.crash(); - } - }, - else => unreachable, // handled above - } - } else { - if (entry.kind != .file) continue; - real_file_count += 1; - const openFile = std.fs.Dir.openFile; - const createFile = std.fs.Dir.createFile; + return real_file_count; + } + }; + + var pkg_in_cache_dir = try cache_dir.openDir(cache_dir_subpath, .{ .iterate = true }); + defer pkg_in_cache_dir.close(); + var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); + defer walker.deinit(); + _ = try FileCopier.copy( + node_modules_folder, + &walker, + {}, + {}, + {}, + {}, + ); + } - var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); - defer in_file.close(); + fn overwriteNodeModulesFolderWindows( + manager: *PackageManager, + cache_dir: std.fs.Dir, + cache_dir_subpath: []const u8, + node_modules_folder_path: []const u8, + parent_dir: std.fs.Dir, + subpath: []const u8, + ) !void { + var node_fs_for_package_installer: bun.JSC.Node.NodeFS = .{}; - if (bun.sys.unlinkat( - bun.toFD(destination_dir_.fd), - entry.basename.ptr[0..entry.basename.len :0], - ).asErr()) |e| { - std.debug.print("unlink problem uh oh {}\n", .{e}); - @panic("TODO zack handle"); - } + var node_modules_folder = try std.fs.cwd().openDir(node_modules_folder_path, .{ .iterate = true }); + defer node_modules_folder.close(); - const mode = try in_file.mode(); - var outfile = try createFile(destination_dir_, entry.basename, .{ .mode = mode }); - defer outfile.close(); - - debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); - // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { - // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { - // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; - // } - // break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { - // if (do_progress) { - // progress_.root.end(); - // progress_.refresh(); - // } - - // Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - // Global.crash(); - // }; - // }; - // defer outfile.close(); + const IGNORED_PATHS: []const bun.OSPathSlice = &[_][]const bun.OSPathChar{ + bun.OSPathLiteral("node_modules"), + bun.OSPathLiteral(".git"), + bun.OSPathLiteral("CMakeFiles"), + }; - if (comptime Environment.isPosix) { - const stat = in_file.stat() catch continue; - _ = C.fchmod(outfile.handle, @intCast(stat.mode)); - } + const FileCopier = struct { + pub fn copy( + destination_dir_: std.fs.Dir, + walker: *Walker, + to_copy_into1: []u16, + head1: []u16, + to_copy_into2: []u16, + head2: []u16, + ) !u32 { + const real_file_count: u32 = 0; + while (try walker.next()) |entry| { + switch (entry.kind) { + .directory, .file => {}, + else => continue, + } - bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { - Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; + if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { + return error.NameTooLong; + } + + @memcpy(to_copy_into1[0..entry.path.len], entry.path); + head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; + const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; + + @memcpy(to_copy_into2[0..entry.path.len], entry.path); + head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; + const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; + + switch (entry.kind) { + .directory => { + if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { + bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; + } + }, + .file => { + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) == 0) { + if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { + bun.MakePath.makePath(u16, destination_dir_, entry_dirname) catch {}; + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) != 0) { + continue; + } + } + + if (bun.windows.Win32Error.get().toSystemErrno()) |err| { + Output.prettyError("{s}: copying file {}", .{ @tagName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + } else { + Output.prettyError("error copying file {}", .{bun.fmt.fmtOSPath(entry.path, .{})}); + } + + Global.crash(); + } + }, + else => unreachable, // handled above } } @@ -10468,13 +10539,69 @@ pub const PackageManager = struct { defer pkg_in_cache_dir.close(); var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); defer walker.deinit(); + + const destbase = parent_dir; + const destpath = subpath; + var buf: bun.windows.WPathBuffer = undefined; + var buf2: bun.windows.WPathBuffer = undefined; + + const cache_dir_subpathZ = manager.allocator.dupeZ(u8, cache_dir_subpath) catch bun.outOfMemory(); + defer manager.allocator.free(cache_dir_subpathZ); + var cached_package_dir = bun.openDir(cache_dir, cache_dir_subpathZ) catch |err| { + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + }; + defer cached_package_dir.close(); + + const dest_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(destbase.fd, &buf, buf.len, 0); + if (dest_path_length == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + + var i: usize = dest_path_length; + if (buf[i] != '\\') { + buf[i] = '\\'; + i += 1; + } + + i += bun.strings.toWPathNormalized(buf[i..], destpath).len; + buf[i] = std.fs.path.sep_windows; + i += 1; + buf[i] = 0; + const fullpath = buf[0..i :0]; + + _ = node_fs_for_package_installer.mkdirRecursiveOSPathImpl(void, {}, fullpath, 0, false).unwrap() catch |err| { + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + }; + const to_copy_buf = buf[fullpath.len..]; + + const cache_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(cached_package_dir.fd, &buf2, buf2.len, 0); + if (cache_path_length == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + const cache_path = buf2[0..cache_path_length]; + var to_copy_buf2: []u16 = undefined; + if (buf2[cache_path.len - 1] != '\\') { + buf2[cache_path.len] = '\\'; + to_copy_buf2 = buf2[cache_path.len + 1 ..]; + } else { + to_copy_buf2 = buf2[cache_path.len..]; + } + _ = try FileCopier.copy( node_modules_folder, &walker, - {}, - {}, - {}, - {}, + to_copy_buf, + &buf, + to_copy_buf2, + &buf2, ); } @@ -10833,8 +10960,6 @@ pub const PackageManager = struct { } } - std.debug.print("OLD FOLDER: {s}\n", .{old_folder}); - std.debug.print("NEW FOLDER: {s}\n", .{new_folder}); const contents = switch (bun.patch.gitDiff(manager.allocator, old_folder, new_folder) catch |e| { Output.prettyError( "error: failed to make diff {s}\n", @@ -11575,10 +11700,6 @@ pub const PackageManager = struct { break :brk ""; } else std.fmt.bufPrint(&resolution_buf, "{}", .{resolution.fmt(buf, .posix)}) catch unreachable; - // if (std.mem.eql(u8, "is-even", name)) { - // std.debug.print("HELLO!\n", .{}); - // } - const patch_patch, const patch_contents_hash, const patch_name_and_version_hash, const remove_patch = brk: { if (this.manager.lockfile.patched_dependencies.entries.len == 0 and this.manager.patched_dependencies_to_remove.entries.len == 0) break :brk .{ null, null, null, false }; var sfb = std.heap.stackFallback(1024, this.lockfile.allocator); diff --git a/src/sys.zig b/src/sys.zig index c4379194da07e1..83ae0765810b6a 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -1699,9 +1699,12 @@ pub fn renameatConcurrently(from_dir_fd: bun.FileDescriptor, from: [:0]const u8, // Fallback path: the folder exists in the cache dir, it might be in a strange state // let's attempt to atomically replace it with the temporary folder's version - if (switch (err.getErrno()) { + if (if (comptime bun.Environment.isPosix) switch (err.getErrno()) { .EXIST, .NOTEMPTY, .OPNOTSUPP => true, else => false, + } else switch (err.getErrno()) { + .EXIST, .NOTEMPTY => true, + else => false, }) { did_atomically_replace = true; switch (bun.sys.renameat2(from_dir_fd, from, to_dir_fd, to, .{ From ce9adc1e0f65b887198851956ce60bea357afb85 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 23:12:25 -0700 Subject: [PATCH 16/51] use bun proc for git diff --- src/install/install.zig | 46 ++++++- src/patch.zig | 285 +++++++++++++++++++++++++++++++++++++++- src/sys.zig | 11 ++ 3 files changed, 330 insertions(+), 12 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 37ea1095583836..b9628984bbbeb1 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10449,10 +10449,6 @@ pub const PackageManager = struct { _ = try FileCopier.copy( node_modules_folder, &walker, - {}, - {}, - {}, - {}, ); } @@ -10960,7 +10956,47 @@ pub const PackageManager = struct { } } - const contents = switch (bun.patch.gitDiff(manager.allocator, old_folder, new_folder) catch |e| { + var cwdbuf: bun.PathBuffer = undefined; + const cwd = switch (bun.sys.getcwdZ(&cwdbuf)) { + .result => |fd| fd, + .err => |e| { + Output.prettyError( + "error: failed to get cwd path {}\n", + .{e}, + ); + Output.flush(); + Global.crash(); + }, + }; + var gitbuf: bun.PathBuffer = undefined; + const git = bun.which(&gitbuf, bun.getenvZ("PATH") orelse "", cwd, "git") orelse { + Output.prettyError( + "error: git must be installed to use `bun patch --commit` \n", + .{}, + ); + Output.flush(); + Global.crash(); + }; + const subproc = bun.patch.Subproc.init( + manager, + old_folder, + new_folder, + cwd[0..cwd.len :0], + git, + ); + + subproc.spawn() catch |e| { + Output.prettyError( + "error: failed to make git dif: {s} \n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }; + + manager.sleepUntil(subproc, bun.patch.Subproc.isDone); + + const contents = switch (bun.patch.gitDiffInternal(manager.allocator, old_folder, new_folder) catch |e| { Output.prettyError( "error: failed to make diff {s}\n", .{@errorName(e)}, diff --git a/src/patch.zig b/src/patch.zig index f0c187dafd8acc..2a191f78faf7ba 100644 --- a/src/patch.zig +++ b/src/patch.zig @@ -1,3 +1,5 @@ +const Output = bun.Output; +const Global = bun.Global; const std = @import("std"); const bun = @import("root").bun; const JSC = bun.JSC; @@ -1113,7 +1115,7 @@ pub const TestingAPIs = struct { const new_folder = new_folder_bunstr.toUTF8(bun.default_allocator); defer new_folder.deinit(); - return switch (gitDiff(bun.default_allocator, old_folder.slice(), new_folder.slice()) catch |e| { + return switch (gitDiffInternal(bun.default_allocator, old_folder.slice(), new_folder.slice()) catch |e| { globalThis.throwError(e, "failed to make diff"); return .undefined; }) { @@ -1232,25 +1234,294 @@ pub const TestingAPIs = struct { } }; -pub fn gitDiff( +pub const Subproc = struct { + const Process = bun.spawn.Process; + const PackageManager = bun.install.PackageManager; + const uv = bun.windows.libuv; + pub const OutputReader = bun.io.BufferedReader; + + manager: *PackageManager, + process: ?*Process = null, + stdout: OutputReader = OutputReader.init(@This()), + stderr: OutputReader = OutputReader.init(@This()), + envp: [:null]?[*:0]const u8, + argv: [:null]?[*:0]const u8, + + is_done: bool = false, + + cwd: [:0]const u8, + old_folder: [:0]const u8, + new_folder: [:0]const u8, + + const ARGV = &[_][:0]const u8{ + "git", + "-c", + "core.safecrlf=false", + "diff", + "--src-prefix=a/", + "--dst-prefix=b/", + "--ignore-cr-at-eol", + "--irreversible-delete", + "--full-index", + "--no-index", + }; + + pub usingnamespace bun.New(@This()); + + pub fn resetPolls(this: *Subproc) void { + if (this.process) |process| { + this.process = null; + process.close(); + process.deref(); + } + + this.stdout.deinit(); + this.stderr.deinit(); + this.stdout = OutputReader.init(@This()); + this.stderr = OutputReader.init(@This()); + } + + pub fn deinit(this: *Subproc) void { + this.resetPolls(); + + this.stdout.deinit(); + this.stderr.deinit(); + + bun.default_allocator.free(this.old_folder); + bun.default_allocator.free(this.new_folder); + + bun.default_allocator.free(this.envp); + bun.default_allocator.free(this.argv); + + this.destroy(); + } + + pub fn isDone(this: *Subproc) bool { + return this.is_done; + } + + pub fn init( + manager: *PackageManager, + old_folder_: []const u8, + new_folder_: []const u8, + cwd: [:0]const u8, + git: [:0]const u8, + ) *Subproc { + const paths = gitDiffPreprocessPaths(bun.default_allocator, old_folder_, new_folder_, true); + + const giiit = bun.default_allocator.dupeZ(u8, git) catch bun.outOfMemory(); + const argv: [:null]?[*:0]const u8 = brk: { + const argv_buf = bun.default_allocator.allocSentinel(?[*:0]const u8, ARGV.len + 2, null) catch bun.outOfMemory(); + argv_buf[0] = giiit.ptr; + for (1..ARGV.len) |i| { + argv_buf[i] = ARGV[i].ptr; + } + argv_buf[ARGV.len] = paths[0]; + argv_buf[ARGV.len + 1] = paths[1]; + break :brk argv_buf; + }; + + const envp: [:null]?[*:0]const u8 = brk: { + const env_arr = &[_][:0]const u8{ + "GIT_CONFIG_NOSYSTEM", + "HOME", + "XDG_CONFIG_HOME", + "USERPROFILE", + }; + const PATH = bun.getenvZ("PATH"); + const envp_buf = bun.default_allocator.allocSentinel(?[*:0]const u8, env_arr.len + @as(usize, if (PATH != null) 1 else 0), null) catch bun.outOfMemory(); + for (0..env_arr.len) |i| { + envp_buf[i] = env_arr[i].ptr; + } + if (PATH) |p| { + envp_buf[envp_buf.len - 1] = @ptrCast(p.ptr); + } + break :brk envp_buf; + }; + + return Subproc.new(Subproc{ + .old_folder = paths[0], + .new_folder = paths[1], + .cwd = cwd, + .argv = argv, + .envp = envp, + .manager = manager, + }); + } + + pub fn loop(this: *const Subproc) *bun.uws.Loop { + return this.manager.event_loop.loop(); + } + + pub fn eventLoop(this: *const Subproc) *JSC.AnyEventLoop { + return &this.manager.event_loop; + } + + pub fn onReaderError(this: *Subproc, err: bun.sys.Error) void { + Output.prettyErrorln("error: Failed to git diff due to error {s}", .{ + @tagName(err.getErrno()), + }); + Output.flush(); + this.maybeFinished(); + } + + pub fn onReaderDone(this: *Subproc) void { + this.maybeFinished(); + } + + fn maybeFinished(this: *Subproc) void { + const process = this.process orelse return; + + this.handleExit(process.status); + } + + fn handleExit(this: *Subproc, status: bun.spawn.Status) void { + defer this.is_done = true; + + switch (status) { + // we can't rely on the exit code because git diff returns 1 even when + // it succeeds + .exited => {}, + .signaled => |signal| { + Output.prettyError( + "error: git diff terminated with{}\n", + .{bun.SignalCode.from(signal).fmt(Output.enable_ansi_colors_stderr)}, + ); + Output.flush(); + Global.crash(); + }, + .err => |err| { + Output.prettyError( + "error: failed to make diff {}\n", + .{err}, + ); + Output.flush(); + Global.crash(); + }, + else => {}, + } + } + + pub fn spawn(this: *Subproc) !void { + if (bun.Environment.isWindows) { + this.stdout.source = .{ .pipe = bun.default_allocator.create(uv.Pipe) catch bun.outOfMemory() }; + this.stderr.source = .{ .pipe = bun.default_allocator.create(uv.Pipe) catch bun.outOfMemory() }; + } + + const spawn_options = bun.spawn.SpawnOptions{ + .stdin = .ignore, + .stdout = if (bun.Environment.isPosix) .buffer else .{ .buffer = this.stdout.source.?.pipe }, + .stderr = if (bun.Environment.isPosix) .buffer else .{ .buffer = this.stdout.source.?.pipe }, + + .cwd = this.cwd, + + .windows = if (bun.Environment.isWindows) + .{ .loop = JSC.EventLoopHandle.init(&this.manager.event_loop) } + else {}, + + .stream = false, + }; + + var spawned = try (try bun.spawn.spawnProcess(&spawn_options, this.argv, this.envp)).unwrap(); + + if (comptime bun.Environment.isPosix) { + if (spawned.stdout) |stdout| { + if (!spawned.memfds[1]) { + this.stdout.setParent(this); + _ = bun.sys.setNonblocking(stdout); + try this.stdout.start(stdout, true).unwrap(); + } else { + this.stdout.setParent(this); + this.stdout.startMemfd(stdout); + } + } + if (spawned.stderr) |stderr| { + if (!spawned.memfds[2]) { + this.stderr.setParent(this); + _ = bun.sys.setNonblocking(stderr); + try this.stderr.start(stderr, true).unwrap(); + } else { + this.stderr.setParent(this); + this.stderr.startMemfd(stderr); + } + } + } else if (comptime bun.Environment.isWindows) { + if (spawned.stdout == .buffer) { + this.stdout.parent = this; + try this.stdout.startWithCurrentPipe().unwrap(); + } + if (spawned.stderr == .buffer) { + this.stderr.parent = this; + try this.stderr.startWithCurrentPipe().unwrap(); + } + } + + const event_loop = &this.manager.event_loop; + var process = spawned.toProcess( + event_loop, + false, + ); + + this.process = process; + switch (process.watchOrReap()) { + .err => |err| { + if (!process.hasExited()) + process.onExit(.{ .err = err }, &std.mem.zeroes(bun.spawn.Rusage)); + }, + .result => {}, + } + } +}; + +fn gitDiffPreprocessPaths( allocator: std.mem.Allocator, old_folder_: []const u8, new_folder_: []const u8, -) !bun.JSC.Node.Maybe(std.ArrayList(u8), std.ArrayList(u8)) { - const old_folder: []const u8 = if (comptime bun.Environment.isWindows) brk: { + comptime sentinel: bool, +) [2]if (sentinel) [:0]const u8 else []const u8 { + const T = if (sentinel) [:0]const u8 else []const u8; + const bump = if (sentinel) 1 else 0; + const old_folder: T = if (comptime bun.Environment.isWindows) brk: { // backslash in the path fucks everything up - const cpy = allocator.alloc(u8, old_folder_.len) catch bun.outOfMemory(); + const cpy = allocator.alloc(u8, old_folder_.len + bump) catch bun.outOfMemory(); @memcpy(cpy, old_folder_); std.mem.replaceScalar(u8, cpy, '\\', '/'); + if (sentinel) { + cpy[old_folder_.len] = 0; + break :brk cpy[0..old_folder_.len :0]; + } break :brk cpy; } else old_folder_; - const new_folder = if (comptime bun.Environment.isWindows) brk: { - const cpy = allocator.alloc(u8, new_folder_.len) catch bun.outOfMemory(); + const new_folder: T = if (comptime bun.Environment.isWindows) brk: { + const cpy = allocator.alloc(u8, new_folder_.len + bump) catch bun.outOfMemory(); @memcpy(cpy, new_folder_); std.mem.replaceScalar(u8, cpy, '\\', '/'); + if (sentinel) { + cpy[new_folder_.len] = 0; + break :brk cpy[0..new_folder_.len :0]; + } break :brk cpy; } else new_folder_; + if (bun.Environment.isPosix and sentinel) { + return .{ + allocator.dupeZ(u8, old_folder) catch bun.outOfMemory(), + allocator.dupeZ(u8, new_folder) catch bun.outOfMemory(), + }; + } + + return .{ old_folder, new_folder }; +} + +pub fn gitDiffInternal( + allocator: std.mem.Allocator, + old_folder_: []const u8, + new_folder_: []const u8, +) !bun.JSC.Node.Maybe(std.ArrayList(u8), std.ArrayList(u8)) { + const paths = gitDiffPreprocessPaths(allocator, old_folder_, new_folder_, false); + const old_folder = paths[0]; + const new_folder = paths[1]; + defer if (comptime bun.Environment.isWindows) { allocator.free(old_folder); allocator.free(new_folder); diff --git a/src/sys.zig b/src/sys.zig index 83ae0765810b6a..09c3cc11ba8271 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -352,6 +352,17 @@ pub fn getcwd(buf: *bun.PathBuffer) Maybe([]const u8) { Result.errnoSys(0, .getcwd).?; } +pub fn getcwdZ(buf: *bun.PathBuffer) Maybe([:0]const u8) { + const Result = Maybe([:0]const u8); + buf[0] = 0; + buf[buf.len - 1] = 0; + const rc: ?[*:0]u8 = @ptrCast(std.c.getcwd(buf, bun.MAX_PATH_BYTES)); + return if (rc != null) + Result{ .result = rc.?[0..std.mem.len(rc.?) :0] } + else + Result.errnoSys(0, .getcwd).?; +} + pub fn fchmod(fd: bun.FileDescriptor, mode: bun.Mode) Maybe(void) { if (comptime Environment.isWindows) { return sys_uv.fchmod(fd, mode); From 9f1443352b9cb1428ec929c5730c8f7210d3f4d8 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Thu, 13 Jun 2024 23:54:45 -0700 Subject: [PATCH 17/51] changes --- src/install/install.zig | 3 ++- src/patch.zig | 34 +++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index b9628984bbbeb1..9b0f23858ee9aa 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10993,10 +10993,11 @@ pub const PackageManager = struct { Output.flush(); Global.crash(); }; + defer subproc.deinit(); manager.sleepUntil(subproc, bun.patch.Subproc.isDone); - const contents = switch (bun.patch.gitDiffInternal(manager.allocator, old_folder, new_folder) catch |e| { + const contents = switch (subproc.diffPostProcess() catch |e| { Output.prettyError( "error: failed to make diff {s}\n", .{@errorName(e)}, diff --git a/src/patch.zig b/src/patch.zig index 2a191f78faf7ba..556033f09c056a 100644 --- a/src/patch.zig +++ b/src/patch.zig @@ -1300,6 +1300,28 @@ pub const Subproc = struct { return this.is_done; } + pub fn diffPostProcess(this: *Subproc) !bun.JSC.Node.Maybe(std.ArrayList(u8), std.ArrayList(u8)) { + var stdout = this.stdout.takeBuffer(); + var stderr = this.stdout.takeBuffer(); + + var deinit_stdout = true; + var deinit_stderr = true; + defer { + if (deinit_stdout) stdout.deinit(); + if (deinit_stderr) stderr.deinit(); + } + + if (stderr.items.len > 0) { + deinit_stderr = false; + return .{ .err = stderr }; + } + + debug("Before postprocess: {s}\n", .{stdout.items}); + try gitDiffPostprocess(&stdout, this.old_folder, this.new_folder); + deinit_stdout = false; + return .{ .result = stdout }; + } + pub fn init( manager: *PackageManager, old_folder_: []const u8, @@ -1403,6 +1425,9 @@ pub const Subproc = struct { } pub fn spawn(this: *Subproc) !void { + this.stdout.setParent(this); + this.stderr.setParent(this); + if (bun.Environment.isWindows) { this.stdout.source = .{ .pipe = bun.default_allocator.create(uv.Pipe) catch bun.outOfMemory() }; this.stderr.source = .{ .pipe = bun.default_allocator.create(uv.Pipe) catch bun.outOfMemory() }; @@ -1479,12 +1504,11 @@ fn gitDiffPreprocessPaths( new_folder_: []const u8, comptime sentinel: bool, ) [2]if (sentinel) [:0]const u8 else []const u8 { - const T = if (sentinel) [:0]const u8 else []const u8; const bump = if (sentinel) 1 else 0; - const old_folder: T = if (comptime bun.Environment.isWindows) brk: { + const old_folder = if (comptime bun.Environment.isWindows) brk: { // backslash in the path fucks everything up const cpy = allocator.alloc(u8, old_folder_.len + bump) catch bun.outOfMemory(); - @memcpy(cpy, old_folder_); + @memcpy(cpy[0..old_folder_.len], old_folder_); std.mem.replaceScalar(u8, cpy, '\\', '/'); if (sentinel) { cpy[old_folder_.len] = 0; @@ -1492,9 +1516,9 @@ fn gitDiffPreprocessPaths( } break :brk cpy; } else old_folder_; - const new_folder: T = if (comptime bun.Environment.isWindows) brk: { + const new_folder = if (comptime bun.Environment.isWindows) brk: { const cpy = allocator.alloc(u8, new_folder_.len + bump) catch bun.outOfMemory(); - @memcpy(cpy, new_folder_); + @memcpy(cpy[0..new_folder_.len], new_folder_); std.mem.replaceScalar(u8, cpy, '\\', '/'); if (sentinel) { cpy[new_folder_.len] = 0; From 64434b00a165c05f16d405b961661142dd9223d1 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Fri, 14 Jun 2024 00:39:05 -0700 Subject: [PATCH 18/51] spawn sync way more ez --- src/install/install.zig | 28 +++++++------ src/patch.zig | 88 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 9b0f23858ee9aa..8df2e0a0ac892b 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10977,27 +10977,29 @@ pub const PackageManager = struct { Output.flush(); Global.crash(); }; - const subproc = bun.patch.Subproc.init( - manager, - old_folder, - new_folder, - cwd[0..cwd.len :0], - git, - ); + const paths = bun.patch.gitDiffPreprocessPaths(bun.default_allocator, old_folder, new_folder, false); + const opts = bun.patch.spawnOpts(paths[0], paths[1], cwd, git, &manager.event_loop); - subproc.spawn() catch |e| { + var spawn_result = switch (bun.spawnSync(&opts) catch |e| { Output.prettyError( - "error: failed to make git dif: {s} \n", + "error: failed to make diff {s}\n", .{@errorName(e)}, ); Output.flush(); Global.crash(); + }) { + .result => |r| r, + .err => |e| { + Output.prettyError( + "error: failed to make diff {}\n", + .{e}, + ); + Output.flush(); + Global.crash(); + }, }; - defer subproc.deinit(); - - manager.sleepUntil(subproc, bun.patch.Subproc.isDone); - const contents = switch (subproc.diffPostProcess() catch |e| { + const contents = switch (bun.patch.diffPostProcess(&spawn_result, paths[0], paths[1]) catch |e| { Output.prettyError( "error: failed to make diff {s}\n", .{@errorName(e)}, diff --git a/src/patch.zig b/src/patch.zig index 556033f09c056a..9ca3c53b0cdf7b 100644 --- a/src/patch.zig +++ b/src/patch.zig @@ -1234,6 +1234,92 @@ pub const TestingAPIs = struct { } }; +pub fn spawnOpts( + old_folder: []const u8, + new_folder: []const u8, + cwd: [:0]const u8, + git: [:0]const u8, + loop: *JSC.AnyEventLoop, +) bun.spawn.sync.Options { + const argv: []const []const u8 = brk: { + const ARGV = &[_][:0]const u8{ + "git", + "-c", + "core.safecrlf=false", + "diff", + "--src-prefix=a/", + "--dst-prefix=b/", + "--ignore-cr-at-eol", + "--irreversible-delete", + "--full-index", + "--no-index", + }; + const argv_buf = bun.default_allocator.alloc([]const u8, ARGV.len + 2) catch bun.outOfMemory(); + argv_buf[0] = git; + for (1..ARGV.len) |i| { + argv_buf[i] = ARGV[i]; + } + argv_buf[ARGV.len] = old_folder; + argv_buf[ARGV.len + 1] = new_folder; + break :brk argv_buf; + }; + + const envp: [:null]?[*:0]const u8 = brk: { + const env_arr = &[_][:0]const u8{ + "GIT_CONFIG_NOSYSTEM", + "HOME", + "XDG_CONFIG_HOME", + "USERPROFILE", + }; + const PATH = bun.getenvZ("PATH"); + const envp_buf = bun.default_allocator.allocSentinel(?[*:0]const u8, env_arr.len + @as(usize, if (PATH != null) 1 else 0), null) catch bun.outOfMemory(); + for (0..env_arr.len) |i| { + envp_buf[i] = env_arr[i].ptr; + } + if (PATH) |p| { + envp_buf[envp_buf.len - 1] = @ptrCast(p.ptr); + } + break :brk envp_buf; + }; + + return bun.spawn.sync.Options{ + .stdout = .buffer, + .stderr = .buffer, + .cwd = cwd, + .envp = envp, + .argv = argv, + .windows = if (bun.Environment.isWindows) .{ .loop = switch (loop.*) { + .js => |x| .{ .js = x }, + .mini => |*x| .{ .mini = x }, + } } else {}, + }; +} + +pub fn diffPostProcess(result: *bun.spawn.sync.Result, old_folder: []const u8, new_folder: []const u8) !bun.JSC.Node.Maybe(std.ArrayList(u8), std.ArrayList(u8)) { + var stdout = std.ArrayList(u8).init(bun.default_allocator); + var stderr = std.ArrayList(u8).init(bun.default_allocator); + + std.mem.swap(std.ArrayList(u8), &stdout, &result.stdout); + std.mem.swap(std.ArrayList(u8), &stderr, &result.stderr); + + var deinit_stdout = true; + var deinit_stderr = true; + defer { + if (deinit_stdout) stdout.deinit(); + if (deinit_stderr) stderr.deinit(); + } + + if (stderr.items.len > 0) { + deinit_stderr = false; + return .{ .err = stderr }; + } + + debug("Before postprocess: {s}\n", .{stdout.items}); + try gitDiffPostprocess(&stdout, old_folder, new_folder); + deinit_stdout = false; + return .{ .result = stdout }; +} + pub const Subproc = struct { const Process = bun.spawn.Process; const PackageManager = bun.install.PackageManager; @@ -1498,7 +1584,7 @@ pub const Subproc = struct { } }; -fn gitDiffPreprocessPaths( +pub fn gitDiffPreprocessPaths( allocator: std.mem.Allocator, old_folder_: []const u8, new_folder_: []const u8, From bc7f13c37fdaf8c573424bdfdaaa40bb57b64769 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Fri, 14 Jun 2024 01:35:17 -0700 Subject: [PATCH 19/51] okay --- src/install/install.zig | 65 ++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 8df2e0a0ac892b..5e2bbe7ae30224 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10334,12 +10334,10 @@ pub const PackageManager = struct { }; defer parent_dir.close(); - manager.overwriteNodeModulesFolderWindows( + manager.overwriteNodeModulesFolderPosix( cache_dir, cache_dir_subpath, module_folder, - parent_dir, - bun.path.basename(module_folder), ) catch |e| { Output.prettyError( "error: error overwriting folder in node_modules: {s}\n", @@ -10389,28 +10387,32 @@ pub const PackageManager = struct { var copy_file_state: bun.CopyFileState = .{}; + var pathbuf: bun.PathBuffer = undefined; + while (try walker.next()) |entry| { if (entry.kind != .file) continue; real_file_count += 1; const openFile = std.fs.Dir.openFile; const createFile = std.fs.Dir.createFile; - var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); + const basename = bun.strings.fromWPath(pathbuf[0..], entry.basename); + + var in_file = try openFile(entry.dir, basename, .{ .mode = .read_only }); defer in_file.close(); if (bun.sys.unlinkat( bun.toFD(destination_dir_.fd), - entry.basename.ptr[0..entry.basename.len :0], + basename.ptr[0..basename.len :0], ).asErr()) |e| { Output.prettyError("error: copying file {}", .{e}); Global.crash(); } const mode = try in_file.mode(); - var outfile = try createFile(destination_dir_, entry.basename, .{ .mode = mode }); + var outfile = try createFile(destination_dir_, basename, .{ .mode = mode }); defer outfile.close(); - debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); + // debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; @@ -10432,10 +10434,37 @@ pub const PackageManager = struct { _ = C.fchmod(outfile.handle, @intCast(stat.mode)); } - bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { - Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; + if (comptime bun.Environment.isPosix) { + bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; + } else { + var buf1: bun.windows.WPathBuffer = undefined; + var buf2: bun.windows.WPathBuffer = undefined; + + const infile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(in_file.handle, &buf1, buf1.len, 0); + if (infile_path_len == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + buf1[infile_path_len] = 0; + + const outfile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(outfile.handle, &buf2, buf2.len, 0); + if (outfile_path_len == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + buf2[outfile_path_len] = 0; + bun.copyFileWithState(buf1[0..infile_path_len :0], buf2[0..outfile_path_len :0], ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; + } } return real_file_count; @@ -10471,6 +10500,13 @@ pub const PackageManager = struct { bun.OSPathLiteral("CMakeFiles"), }; + const IGNORED_PATHS2: []const []const u8 = &[_][]const u8{ + "node_modules", + ".git", + "CMakeFiles", + }; + _ = IGNORED_PATHS2; // autofix + const FileCopier = struct { pub fn copy( destination_dir_: std.fs.Dir, @@ -10501,11 +10537,12 @@ pub const PackageManager = struct { switch (entry.kind) { .directory => { - if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { - bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; - } + // if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { + // bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; + // } }, .file => { + // bun.sys.unlinkat(dirfd: bun.FileDescriptor, to: anytype) if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) == 0) { if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { bun.MakePath.makePath(u16, destination_dir_, entry_dirname) catch {}; From 11ee415de4e336f137323bac308b68a46c6897d3 Mon Sep 17 00:00:00 2001 From: Zack Radisic Date: Fri, 14 Jun 2024 23:25:07 +0200 Subject: [PATCH 20/51] fix overwriting for packages with multiple directories --- src/install/install.zig | 174 ++--------------------------- test/cli/install/bun-patch.test.ts | 22 ++++ 2 files changed, 31 insertions(+), 165 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 5e2bbe7ae30224..5310159d3232d3 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10363,7 +10363,7 @@ pub const PackageManager = struct { return; } - fn overwriteNodeModulesFolderPosix( + fn overwriteNodeModulesFolder( manager: *PackageManager, cache_dir: std.fs.Dir, cache_dir_subpath: []const u8, @@ -10386,8 +10386,8 @@ pub const PackageManager = struct { var real_file_count: u32 = 0; var copy_file_state: bun.CopyFileState = .{}; - var pathbuf: bun.PathBuffer = undefined; + // _ = pathbuf; // autofix while (try walker.next()) |entry| { if (entry.kind != .file) continue; @@ -10395,21 +10395,22 @@ pub const PackageManager = struct { const openFile = std.fs.Dir.openFile; const createFile = std.fs.Dir.createFile; - const basename = bun.strings.fromWPath(pathbuf[0..], entry.basename); - - var in_file = try openFile(entry.dir, basename, .{ .mode = .read_only }); + var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); defer in_file.close(); + @memcpy(pathbuf[0..entry.path.len], entry.path); + pathbuf[entry.path.len] = 0; + if (bun.sys.unlinkat( bun.toFD(destination_dir_.fd), - basename.ptr[0..basename.len :0], + pathbuf[0..entry.path.len :0], ).asErr()) |e| { - Output.prettyError("error: copying file {}", .{e}); + Output.prettyError("error: copying file {s} {}", .{e.withPath(entry.path)}); Global.crash(); } const mode = try in_file.mode(); - var outfile = try createFile(destination_dir_, basename, .{ .mode = mode }); + var outfile = try createFile(destination_dir_, entry.basename, .{ .mode = mode }); defer outfile.close(); // debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); @@ -10481,163 +10482,6 @@ pub const PackageManager = struct { ); } - fn overwriteNodeModulesFolderWindows( - manager: *PackageManager, - cache_dir: std.fs.Dir, - cache_dir_subpath: []const u8, - node_modules_folder_path: []const u8, - parent_dir: std.fs.Dir, - subpath: []const u8, - ) !void { - var node_fs_for_package_installer: bun.JSC.Node.NodeFS = .{}; - - var node_modules_folder = try std.fs.cwd().openDir(node_modules_folder_path, .{ .iterate = true }); - defer node_modules_folder.close(); - - const IGNORED_PATHS: []const bun.OSPathSlice = &[_][]const bun.OSPathChar{ - bun.OSPathLiteral("node_modules"), - bun.OSPathLiteral(".git"), - bun.OSPathLiteral("CMakeFiles"), - }; - - const IGNORED_PATHS2: []const []const u8 = &[_][]const u8{ - "node_modules", - ".git", - "CMakeFiles", - }; - _ = IGNORED_PATHS2; // autofix - - const FileCopier = struct { - pub fn copy( - destination_dir_: std.fs.Dir, - walker: *Walker, - to_copy_into1: []u16, - head1: []u16, - to_copy_into2: []u16, - head2: []u16, - ) !u32 { - const real_file_count: u32 = 0; - while (try walker.next()) |entry| { - switch (entry.kind) { - .directory, .file => {}, - else => continue, - } - - if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { - return error.NameTooLong; - } - - @memcpy(to_copy_into1[0..entry.path.len], entry.path); - head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; - const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; - - @memcpy(to_copy_into2[0..entry.path.len], entry.path); - head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; - const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; - - switch (entry.kind) { - .directory => { - // if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { - // bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; - // } - }, - .file => { - // bun.sys.unlinkat(dirfd: bun.FileDescriptor, to: anytype) - if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) == 0) { - if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { - bun.MakePath.makePath(u16, destination_dir_, entry_dirname) catch {}; - if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) != 0) { - continue; - } - } - - if (bun.windows.Win32Error.get().toSystemErrno()) |err| { - Output.prettyError("{s}: copying file {}", .{ @tagName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - } else { - Output.prettyError("error copying file {}", .{bun.fmt.fmtOSPath(entry.path, .{})}); - } - - Global.crash(); - } - }, - else => unreachable, // handled above - } - } - - return real_file_count; - } - }; - - var pkg_in_cache_dir = try cache_dir.openDir(cache_dir_subpath, .{ .iterate = true }); - defer pkg_in_cache_dir.close(); - var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); - defer walker.deinit(); - - const destbase = parent_dir; - const destpath = subpath; - var buf: bun.windows.WPathBuffer = undefined; - var buf2: bun.windows.WPathBuffer = undefined; - - const cache_dir_subpathZ = manager.allocator.dupeZ(u8, cache_dir_subpath) catch bun.outOfMemory(); - defer manager.allocator.free(cache_dir_subpathZ); - var cached_package_dir = bun.openDir(cache_dir, cache_dir_subpathZ) catch |err| { - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - }; - defer cached_package_dir.close(); - - const dest_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(destbase.fd, &buf, buf.len, 0); - if (dest_path_length == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - } - - var i: usize = dest_path_length; - if (buf[i] != '\\') { - buf[i] = '\\'; - i += 1; - } - - i += bun.strings.toWPathNormalized(buf[i..], destpath).len; - buf[i] = std.fs.path.sep_windows; - i += 1; - buf[i] = 0; - const fullpath = buf[0..i :0]; - - _ = node_fs_for_package_installer.mkdirRecursiveOSPathImpl(void, {}, fullpath, 0, false).unwrap() catch |err| { - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - }; - const to_copy_buf = buf[fullpath.len..]; - - const cache_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(cached_package_dir.fd, &buf2, buf2.len, 0); - if (cache_path_length == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - } - const cache_path = buf2[0..cache_path_length]; - var to_copy_buf2: []u16 = undefined; - if (buf2[cache_path.len - 1] != '\\') { - buf2[cache_path.len] = '\\'; - to_copy_buf2 = buf2[cache_path.len + 1 ..]; - } else { - to_copy_buf2 = buf2[cache_path.len..]; - } - - _ = try FileCopier.copy( - node_modules_folder, - &walker, - to_copy_buf, - &buf, - to_copy_buf2, - &buf2, - ); - } - const PatchCommitResult = struct { patch_key: []const u8, patchfile_path: []const u8, diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 77c9fc9ce5fed4..0d6ddca14c3da7 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -196,6 +196,28 @@ Once you're done with your changes, run: }); } + test("overwriting module with multiple levels of directories", async () => { + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { lodash: "4.17.21" }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); + + { + const { stderr } = await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } + + { + const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } + }); + ["is-even@1.0.0", "node_modules/is-even"].map(patchArg => makeTest("should patch a node_modules package", { dependencies: { "is-even": "1.0.0" }, From e53230ff8c22d82d3b4afae6eee3a647fd717c19 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Fri, 14 Jun 2024 17:39:07 -0700 Subject: [PATCH 21/51] fix compile for posix --- src/install/install.zig | 70 ++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 5310159d3232d3..049ce76c8d5097 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10322,40 +10322,14 @@ pub const PackageManager = struct { // meaning that changes to the folder will also change the package in the cache. // // So we will overwrite the folder by directly copying the package in cache into it - if (comptime bun.Environment.isWindows) { - const modules_parent_dir = bun.path.dirname(module_folder, .auto); - var parent_dir = std.fs.cwd().openDir(modules_parent_dir, .{ .iterate = true }) catch |e| { - Output.prettyError( - "error: error opening folder in node_modules: {s}\n", - .{@errorName(e)}, - ); - Output.flush(); - Global.crash(); - }; - defer parent_dir.close(); - - manager.overwriteNodeModulesFolderPosix( - cache_dir, - cache_dir_subpath, - module_folder, - ) catch |e| { - Output.prettyError( - "error: error overwriting folder in node_modules: {s}\n", - .{@errorName(e)}, - ); - Output.flush(); - Global.crash(); - }; - } else { - manager.overwriteNodeModulesFolderPosix(cache_dir, cache_dir_subpath, module_folder) catch |e| { - Output.prettyError( - "error: error overwriting folder in node_modules: {s}\n", - .{@errorName(e)}, - ); - Output.flush(); - Global.crash(); - }; - } + manager.overwriteNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { + Output.prettyError( + "error: error overwriting folder in node_modules: {s}\n", + .{@errorName(e)}, + ); + Output.flush(); + Global.crash(); + }; Output.pretty("\nTo patch {s}, edit the following folder:\n\n {s}\n", .{ pkg_name, module_folder }); Output.pretty("\nOnce you're done with your changes, run:\n\n bun patch --commit '{s}'\n", .{module_folder}); @@ -10387,14 +10361,40 @@ pub const PackageManager = struct { var copy_file_state: bun.CopyFileState = .{}; var pathbuf: bun.PathBuffer = undefined; + var pathbuf2: bun.PathBuffer = undefined; // _ = pathbuf; // autofix + // var tmpdir: if (bun.Environment.isWindows) std.fs.Dir else struct {} = if (comptime bun.Environment.isWindows) brk: {} else .{}; + while (try walker.next()) |entry| { if (entry.kind != .file) continue; real_file_count += 1; const openFile = std.fs.Dir.openFile; const createFile = std.fs.Dir.createFile; + // - rename node_modules/$PKG/$FILE -> $TMPDIR/$FILE + // - create node_modules/$PKG/$FILE + // - copy: cache/$PKG/$FILE -> node_modules/$PKG/$FILE + // - unlink: $TMPDIR/$FILE + if (comptime bun.Environment.isWindows) { + const basename = bun.strings.fromWPath(pathbuf2[0..], entry.basename); + var tmpbuf: [1024]u8 = undefined; + const tmpname = bun.span(bun.fs.FileSystem.instance.tmpname(basename, tmpbuf[0..], bun.fastRandom()) catch |e| { + Output.prettyError("error: copying file {s}", .{@errorName(e)}); + Global.crash(); + }); + _ = tmpname; // autofix + + @memcpy(pathbuf[0..entry.path.len], entry.path); + pathbuf[entry.path.len] = 0; + + // bun.sys.renameatConcurrently(bun.toFD(destination_dir_.fd), pathbuf[0..entry.path.len :0], to_dir_fd: bun.FileDescriptor, to: [:0]const u8); + // var in_file = try openFile(entry.dir, basename, .{ .mode = .read_only }); + // defer in_file.close(); + + continue; + } + var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); defer in_file.close(); @@ -10405,7 +10405,7 @@ pub const PackageManager = struct { bun.toFD(destination_dir_.fd), pathbuf[0..entry.path.len :0], ).asErr()) |e| { - Output.prettyError("error: copying file {s} {}", .{e.withPath(entry.path)}); + Output.prettyError("error: copying file {}", .{e.withPath(entry.path)}); Global.crash(); } From 7418d3ac727db8c4085665358d3c2443633dd1ba Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Fri, 14 Jun 2024 21:34:55 -0700 Subject: [PATCH 22/51] make it work on windows --- src/install/install.zig | 263 +++++++++++++++++++++++++++++++++------- 1 file changed, 218 insertions(+), 45 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 049ce76c8d5097..c579b4b8b4258a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10322,7 +10322,7 @@ pub const PackageManager = struct { // meaning that changes to the folder will also change the package in the cache. // // So we will overwrite the folder by directly copying the package in cache into it - manager.overwriteNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { + manager.overwritePackageInNodeModulesFolder(cache_dir, cache_dir_subpath, module_folder) catch |e| { Output.prettyError( "error: error overwriting folder in node_modules: {s}\n", .{@errorName(e)}, @@ -10337,7 +10337,7 @@ pub const PackageManager = struct { return; } - fn overwriteNodeModulesFolder( + fn overwritePackageInNodeModulesFolder( manager: *PackageManager, cache_dir: std.fs.Dir, cache_dir_subpath: []const u8, @@ -10364,33 +10364,73 @@ pub const PackageManager = struct { var pathbuf2: bun.PathBuffer = undefined; // _ = pathbuf; // autofix - // var tmpdir: if (bun.Environment.isWindows) std.fs.Dir else struct {} = if (comptime bun.Environment.isWindows) brk: {} else .{}; - while (try walker.next()) |entry| { if (entry.kind != .file) continue; real_file_count += 1; const openFile = std.fs.Dir.openFile; const createFile = std.fs.Dir.createFile; - // - rename node_modules/$PKG/$FILE -> $TMPDIR/$FILE + // - rename node_modules/$PKG/$FILE -> node_modules/$PKG/$TMPNAME // - create node_modules/$PKG/$FILE // - copy: cache/$PKG/$FILE -> node_modules/$PKG/$FILE // - unlink: $TMPDIR/$FILE if (comptime bun.Environment.isWindows) { - const basename = bun.strings.fromWPath(pathbuf2[0..], entry.basename); var tmpbuf: [1024]u8 = undefined; + const basename = bun.strings.fromWPath(pathbuf2[0..], entry.basename); const tmpname = bun.span(bun.fs.FileSystem.instance.tmpname(basename, tmpbuf[0..], bun.fastRandom()) catch |e| { Output.prettyError("error: copying file {s}", .{@errorName(e)}); Global.crash(); }); - _ = tmpname; // autofix - @memcpy(pathbuf[0..entry.path.len], entry.path); - pathbuf[entry.path.len] = 0; + const entrypath = bun.strings.fromWPath(pathbuf[0..], entry.path); + pathbuf[entrypath.len] = 0; + const entrypathZ = pathbuf[0..entrypath.len :0]; + + if (bun.sys.renameatConcurrently( + bun.toFD(destination_dir_.fd), + entrypathZ, + bun.toFD(destination_dir_.fd), + tmpname, + ).asErr()) |e| { + Output.prettyError("error: copying file {}", .{e}); + Global.crash(); + } + + var in_file = openFile(entry.dir, basename, .{ .mode = .read_only }) catch |e| { + Output.prettyError("error: opening node_modules file {s} {s} ()", .{ @errorName(e), entrypath }); + Global.crash(); + }; + defer in_file.close(); + + const mode = in_file.mode() catch @panic("OH NO"); + var outfile = createFile(destination_dir_, entrypath, .{ .mode = mode }) catch @panic("OH NO"); + defer outfile.close(); - // bun.sys.renameatConcurrently(bun.toFD(destination_dir_.fd), pathbuf[0..entry.path.len :0], to_dir_fd: bun.FileDescriptor, to: [:0]const u8); - // var in_file = try openFile(entry.dir, basename, .{ .mode = .read_only }); - // defer in_file.close(); + var buf1: bun.windows.WPathBuffer = undefined; + var buf2: bun.windows.WPathBuffer = undefined; + + const infile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(in_file.handle, &buf1, buf1.len, 0); + if (infile_path_len == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + buf1[infile_path_len] = 0; + + const outfile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(outfile.handle, &buf2, buf2.len, 0); + if (outfile_path_len == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + buf2[outfile_path_len] = 0; + + bun.copyFileWithState(buf1[0..infile_path_len :0], buf2[0..outfile_path_len :0], ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; continue; } @@ -10409,8 +10449,7 @@ pub const PackageManager = struct { Global.crash(); } - const mode = try in_file.mode(); - var outfile = try createFile(destination_dir_, entry.basename, .{ .mode = mode }); + var outfile = try createFile(destination_dir_, entry.path, .{}); defer outfile.close(); // debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); @@ -10435,37 +10474,10 @@ pub const PackageManager = struct { _ = C.fchmod(outfile.handle, @intCast(stat.mode)); } - if (comptime bun.Environment.isPosix) { - bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { - Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; - } else { - var buf1: bun.windows.WPathBuffer = undefined; - var buf2: bun.windows.WPathBuffer = undefined; - - const infile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(in_file.handle, &buf1, buf1.len, 0); - if (infile_path_len == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - } - buf1[infile_path_len] = 0; - - const outfile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(outfile.handle, &buf2, buf2.len, 0); - if (outfile_path_len == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - } - buf2[outfile_path_len] = 0; - bun.copyFileWithState(buf1[0..infile_path_len :0], buf2[0..outfile_path_len :0], ©_file_state) catch |err| { - Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; - } + bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; } return real_file_count; @@ -10482,6 +10494,167 @@ pub const PackageManager = struct { ); } + // fn overwriteNodeModulesFolder( + // manager: *PackageManager, + // cache_dir: std.fs.Dir, + // cache_dir_subpath: []const u8, + // node_modules_folder_path: []const u8, + // ) !void { + // var node_modules_folder = try std.fs.cwd().openDir(node_modules_folder_path, .{ .iterate = true }); + // defer node_modules_folder.close(); + + // const IGNORED_PATHS: []const bun.OSPathSlice = &[_][]const bun.OSPathChar{ + // bun.OSPathLiteral("node_modules"), + // bun.OSPathLiteral(".git"), + // bun.OSPathLiteral("CMakeFiles"), + // }; + + // const FileCopier = struct { + // pub fn copy( + // destination_dir_: std.fs.Dir, + // walker: *Walker, + // ) !u32 { + // var real_file_count: u32 = 0; + + // var copy_file_state: bun.CopyFileState = .{}; + // var pathbuf: bun.PathBuffer = undefined; + // var pathbuf2: bun.PathBuffer = undefined; + // // _ = pathbuf; // autofix + + // while (try walker.next()) |entry| { + // if (entry.kind != .file) continue; + // real_file_count += 1; + // const openFile = std.fs.Dir.openFile; + // const createFile = std.fs.Dir.createFile; + + // // - rename node_modules/$PKG/$FILE -> node_modules/$PKG/$TMPNAME + // // - create node_modules/$PKG/$FILE + // // - copy: cache/$PKG/$FILE -> node_modules/$PKG/$FILE + // // - unlink: $TMPDIR/$FILE + // if (comptime bun.Environment.isWindows) { + // var tmpbuf: [1024]u8 = undefined; + // const basename = bun.strings.fromWPath(pathbuf2[0..], entry.basename); + // const tmpname = bun.span(bun.fs.FileSystem.instance.tmpname(basename, tmpbuf[0..], bun.fastRandom()) catch |e| { + // Output.prettyError("error: copying file {s}", .{@errorName(e)}); + // Global.crash(); + // }); + + // const entrypath = bun.strings.fromWPath(pathbuf[0..], entry.path); + // pathbuf[entrypath.len] = 0; + // const entrypathZ = pathbuf[0..entrypath.len :0]; + + // if (bun.sys.renameatConcurrently( + // bun.toFD(destination_dir_.fd), + // entrypathZ, + // bun.toFD(destination_dir_.fd), + // tmpname, + // ).asErr()) |e| { + // Output.prettyError("error: copying file {}", .{e}); + // Global.crash(); + // } + + // var in_file = try openFile(entry.dir, entrypath, .{ .mode = .read_only }); + // defer in_file.close(); + + // if (bun.sys.unlinkat( + // bun.toFD(destination_dir_.fd), + // entrypath, + // ).asErr()) |e| { + // Output.prettyError("error: copying file {}", .{e.withPath(entrypath)}); + // Global.crash(); + // } + + // const mode = try in_file.mode(); + // var outfile = try createFile(destination_dir_, entrypath, .{ .mode = mode }); + // defer outfile.close(); + + // var buf1: bun.windows.WPathBuffer = undefined; + // var buf2: bun.windows.WPathBuffer = undefined; + + // const infile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(in_file.handle, &buf1, buf1.len, 0); + // if (infile_path_len == 0) { + // const e = bun.windows.Win32Error.get(); + // const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + // Output.prettyError("error: copying in path {}", .{err}); + // Global.crash(); + // } + // buf1[infile_path_len] = 0; + + // const outfile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(outfile.handle, &buf2, buf2.len, 0); + // if (outfile_path_len == 0) { + // const e = bun.windows.Win32Error.get(); + // const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + // Output.prettyError("error: getting out path {}", .{err}); + // Global.crash(); + // } + // buf2[outfile_path_len] = 0; + // bun.copyFileWithState(buf1[0..infile_path_len :0], buf2[0..outfile_path_len :0], ©_file_state) catch |err| { + // Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + // Global.crash(); + // }; + + // continue; + // } + + // var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); + // defer in_file.close(); + + // @memcpy(pathbuf[0..entry.path.len], entry.path); + // pathbuf[entry.path.len] = 0; + + // if (bun.sys.unlinkat( + // bun.toFD(destination_dir_.fd), + // pathbuf[0..entry.path.len :0], + // ).asErr()) |e| { + // Output.prettyError("error: copying file {}", .{e.withPath(entry.path)}); + // Global.crash(); + // } + + // var outfile = try createFile(destination_dir_, entry.path, .{}); + // defer outfile.close(); + + // // debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); + // // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { + // // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { + // // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; + // // } + // // break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { + // // if (do_progress) { + // // progress_.root.end(); + // // progress_.refresh(); + // // } + + // // Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + // // Global.crash(); + // // }; + // // }; + // // defer outfile.close(); + + // if (comptime Environment.isPosix) { + // const stat = in_file.stat() catch continue; + // _ = C.fchmod(outfile.handle, @intCast(stat.mode)); + // } + + // bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { + // Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + // Global.crash(); + // }; + // } + + // return real_file_count; + // } + // }; + + // var pkg_in_cache_dir = try cache_dir.openDir(cache_dir_subpath, .{ .iterate = true }); + // defer pkg_in_cache_dir.close(); + // var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); + // defer walker.deinit(); + // _ = try FileCopier.copy( + // node_modules_folder, + // &walker, + // ); + // } + const PatchCommitResult = struct { patch_key: []const u8, patchfile_path: []const u8, From afc0623d38d00311d16e94e26b4c4706c37f2da6 Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 14 Jun 2024 22:12:32 -0700 Subject: [PATCH 23/51] missing lockfile --- src/install/install.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/install/install.zig b/src/install/install.zig index c579b4b8b4258a..57edc9a9fd3e15 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10675,7 +10675,8 @@ pub const PackageManager = struct { defer lockfile.deinit(); switch (lockfile.loadFromDisk(manager, manager.allocator, manager.log, manager.options.lockfile_path, true)) { .not_found => { - Output.panic("Lockfile not found", .{}); + Output.errGeneric("Cannot find lockfile. Install packages with `bun install` before patching them.", .{}); + Global.crash(); }, .err => |cause| { if (log_level != .silent) { From 8d4b3a662381ac6b8ba0308f675c336f5be0d71f Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Fri, 14 Jun 2024 22:55:36 -0700 Subject: [PATCH 24/51] fix index out of bounds --- src/install/dependency.zig | 15 +++++++++++++++ src/install/install.zig | 27 ++------------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/install/dependency.zig b/src/install/dependency.zig index 5dd1a554b8937f..65c73d5428bcd2 100644 --- a/src/install/dependency.zig +++ b/src/install/dependency.zig @@ -259,6 +259,21 @@ pub inline fn isRemoteTarball(dependency: string) bool { return strings.hasPrefixComptime(dependency, "https://") or strings.hasPrefixComptime(dependency, "http://"); } +/// Turns `foo@1.1.1` into `foo`, `1.1.1`, or `@foo/bar@1.1.1` into `@foo/bar`, `1.1.1`, or `foo` into `foo`, `null`. +pub fn splitNameAndVersion(str: string) struct { string, ?string } { + if (strings.indexOfChar(str, '@')) |at_index| { + if (at_index != 0) { + return .{ str[0..at_index], if (at_index + 1 < str.len) str[at_index + 1 ..] else null }; + } + + const second_at_index = (strings.indexOfChar(str[1..], '@') orelse return .{ str, null }) + 1; + + return .{ str[0..second_at_index], if (second_at_index + 1 < str.len) str[second_at_index + 1 ..] else null }; + } + + return .{ str, null }; +} + pub const Version = struct { tag: Tag = .uninitialized, literal: String = .{}, diff --git a/src/install/install.zig b/src/install/install.zig index 57edc9a9fd3e15..ef175a2f9d6830 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10258,19 +10258,7 @@ pub const PackageManager = struct { }, .name_and_version => brk: { const pkg_maybe_version_to_patch = argument; - const name: []const u8, const version: ?[]const u8 = namever: { - if (std.mem.indexOfScalar(u8, pkg_maybe_version_to_patch[1..], '@')) |version_delimiter| { - break :namever .{ - pkg_maybe_version_to_patch[0 .. version_delimiter + 1], - pkg_maybe_version_to_patch[version_delimiter + 2 ..], - }; - } - break :namever .{ - pkg_maybe_version_to_patch, - null, - }; - }; - + const name, const version = Dependency.splitNameAndVersion(pkg_maybe_version_to_patch); const result = pkg_dep_id_for_name_and_version(manager.lockfile, pkg_maybe_version_to_patch, name, version); const pkg_id = result[0]; const dependency_id = result[1]; @@ -10822,18 +10810,7 @@ pub const PackageManager = struct { break :result .{ cache_dir, cache_dir_subpath, changes_dir, actual_package }; }, .name_and_version => brk: { - const name: []const u8, const version: ?[]const u8 = brk1: { - if (std.mem.indexOfScalar(u8, argument[1..], '@')) |version_delimiter| { - break :brk1 .{ - argument[0 .. version_delimiter + 1], - argument[version_delimiter + 2 ..], - }; - } - break :brk1 .{ - argument, - null, - }; - }; + const name, const version = Dependency.splitNameAndVersion(argument); const result = pkg_dep_id_for_name_and_version(lockfile, argument, name, version); const pkg_id: PackageID = result[0]; const dependency_id: DependencyID = result[1]; From 54cd02c4bb8c88defd0d2651180574a228883919 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 17 Jun 2024 11:43:37 -0700 Subject: [PATCH 25/51] fix unable to find package on windows --- src/install/install.zig | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index ef175a2f9d6830..07c36c02fd16f7 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10132,6 +10132,21 @@ pub const PackageManager = struct { return .{ pkg_id, dependency_id }; } + const PatchArgKind = enum { + path, + name_and_version, + + pub fn fromArg(argument: []const u8) PatchArgKind { + if (bun.strings.hasPrefix(argument, "node_modules/")) return .path; + if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules/")) return .path; + if (comptime bun.Environment.isWindows) { + if (bun.strings.hasPrefix(argument, "node_modules\\")) return .path; + if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules\\")) return .path; + } + return .name_and_version; + } + }; + /// 1. Arg is either: /// - name and possibly version (e.g. "is-even" or "is-even@1.0.0") /// - path to package in node_modules @@ -10142,16 +10157,7 @@ pub const PackageManager = struct { const strbuf = manager.lockfile.buffers.string_bytes.items; const argument = manager.options.positionals[1]; - const ArgKind = enum { - path, - name_and_version, - }; - - const arg_kind: ArgKind = brk: { - if (bun.strings.hasPrefix(argument, "node_modules/")) break :brk .path; - if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules/")) break :brk .path; - break :brk .name_and_version; - }; + const arg_kind: PatchArgKind = PatchArgKind.fromArg(argument); var folder_path_buf: bun.PathBuffer = undefined; var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); @@ -10696,18 +10702,9 @@ pub const PackageManager = struct { .ok => {}, } - const ArgKind = enum { - path, - name_and_version, - }; - const argument = manager.options.positionals[1]; - const arg_kind: ArgKind = brk: { - if (bun.strings.hasPrefix(argument, "node_modules/")) break :brk .path; - if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules/")) break :brk .path; - break :brk .name_and_version; - }; + const arg_kind: PatchArgKind = PatchArgKind.fromArg(argument); // Attempt to open the existing node_modules folder var root_node_modules = switch (bun.sys.openatOSPath(bun.FD.cwd(), bun.OSPathLiteral("node_modules"), std.os.O.DIRECTORY | std.os.O.RDONLY, 0o755)) { From 4fc53a9d83aadf1c056fdcabf01445e88fb6a013 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 11:45:12 -0700 Subject: [PATCH 26/51] run twice --- test/cli/install/bun-patch.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 0d6ddca14c3da7..d48c2e40b4586f 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -216,6 +216,12 @@ Once you're done with your changes, run: const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); expect(stderr.toString()).not.toContain("error"); } + + // run it again to make sure we didn't f something up + { + const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } }); ["is-even@1.0.0", "node_modules/is-even"].map(patchArg => From 3838ec1327cd27dfb1a24f420d683ec9de84e1dd Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:04:18 -0700 Subject: [PATCH 27/51] more tests --- src/install/patch_install.zig | 35 ++++++------- src/sys.zig | 2 + test/cli/install/bun-patch.test.ts | 83 ++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index f53575ff15626b..490a68cd3aadfd 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -457,27 +457,26 @@ pub const PatchTask = struct { // what's a good number for this? page size i guess const STACK_SIZE = 16384; + var file = bun.sys.File{ .handle = fd }; var stack: [STACK_SIZE]u8 = undefined; var read: usize = 0; while (read < size) { - var i: usize = 0; - while (i < STACK_SIZE and i < size) { - switch (bun.sys.read(fd, stack[i..])) { - .result => |w| i += w, - .err => |e| { - log.addErrorFmt( - null, - Loc.Empty, - this.manager.allocator, - "failed to read from patch file: {} ({s})", - .{ e, absolute_patchfile_path }, - ) catch bun.outOfMemory(); - return null; - }, - } - } - read += i; - hasher.update(stack[0..i]); + const slice = switch (file.readFillBuf(stack[0..])) { + .result => |slice| slice, + .err => |e| { + log.addErrorFmt( + null, + Loc.Empty, + this.manager.allocator, + "failed to read from patch file: {} ({s})", + .{ e, absolute_patchfile_path }, + ) catch bun.outOfMemory(); + return null; + }, + }; + if (slice.len == 0) break; + hasher.update(slice); + read += slice.len; } return hasher.final(); diff --git a/src/sys.zig b/src/sys.zig index 9118a3c8d5e653..527592d830b321 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -2956,6 +2956,7 @@ pub const File = struct { return self.bytes.items; } }; + pub fn readFillBuf(this: File, buf: []u8) Maybe([]u8) { var read_amount: usize = 0; while (read_amount < buf.len) { @@ -2978,6 +2979,7 @@ pub const File = struct { return .{ .result = buf[0..read_amount] }; } + pub fn readToEndWithArrayList(this: File, list: *std.ArrayList(u8)) Maybe(usize) { const size = switch (this.getEndPos()) { .err => |err| { diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index d48c2e40b4586f..6d88e416899120 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -3,7 +3,90 @@ import { bunExe, bunEnv as env, toBeValidBin, toHaveBins, toBeWorkspaceLink, tem import { afterAll, afterEach, beforeAll, beforeEach, expect, it, describe, test, setDefaultTimeout } from "bun:test"; import { join, sep } from "path"; +const expectNoError = (o: ShellOutput) => expect(o.stderr.toString()).not.toContain("error"); + describe("bun patch ", async () => { + // Tests to make sure that patching + describe("popular pkg", async () => { + const dummyCode = /* ts */ ` + module.exports = function lmao() { + return 420; + } + `; + + function makeTest(pkgName: string, version: string, folder_in_node_modules: string = `${pkgName}`) { + test( + `${pkgName}@${version}`, + async () => { + const tempdir = tempDirWithFiles("popular", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { + [pkgName]: version, + }, + }), + "index.ts": /* ts */ `import lmao from '${pkgName}'; console.log(lmao())`, + }); + + console.log("TEMPDIR", tempdir); + expectNoError(await $`${bunExe()} i`.env(bunEnv).cwd(tempdir)); + expectNoError(await $`${bunExe()} patch ${pkgName}@${version}`.env(bunEnv).cwd(tempdir)); + await $`echo ${dummyCode} > node_modules/${folder_in_node_modules}/index.js`.env(bunEnv).cwd(tempdir); + const { type, module, exports, ...package_json }: Record = + await $`cat node_modules/${folder_in_node_modules}/package.json`.env(bunEnv).cwd(tempdir).json(); + package_json["main"] = "index.js"; + await $`echo ${JSON.stringify(package_json)} > node_modules/${folder_in_node_modules}/package.json` + .env(bunEnv) + .cwd(tempdir); + + expectNoError( + await $`${bunExe()} patch --commit node_modules/${folder_in_node_modules}`.env(bunEnv).cwd(tempdir), + ); + + const { stdout } = await $`${bunExe()} run index.ts`.env(bunEnv).cwd(tempdir); + expect(stdout.toString()).toBe("420\n"); + }, + 30 * 1000, + ); + } + + makeTest("lodash", "4.17.21"); + makeTest("react", "18.3.1"); + makeTest("react-dom", "18.3.1"); + makeTest("axios", "1.7.2"); + makeTest("tslib", "2.6.3"); + makeTest("chalk", "5.3.0"); + makeTest("next", "14.2.4"); + makeTest("express", "4.19.2"); + makeTest("inquirer", "9.2.23"); + makeTest("commander", "12.1.0"); + + // vercel/next.js + makeTest("webpack-sources", "3.2.3"); + + // vitejs/vite + makeTest("acorn", "8.11.3"); + makeTest("chokidar", "3.6.0"); + makeTest("http-proxy", "1.18.1"); + makeTest("sirv", "2.0.4"); + + // mermaid-js/mermaid + makeTest("cytoscape", "3.28.1"); + + // remix-run/react-router + makeTest("@changesets/get-dependents-graph", "1.3.6", "@changesets/get-dependents-graph"); + + // n8n-io/n8n + makeTest("typedi", "0.10.0"); + makeTest("@sentry/cli", "2.17.0", "@sentry/cli"); + makeTest("pkce-challenge", "3.0.0"); + makeTest("pyodide", "0.23.4"); + makeTest("@types/express-serve-static-core", "4.17.43", "@types/express-serve-static-core"); + makeTest("@types/ws", "8.5.4", "@types/ws"); + makeTest("@types/uuencode", "0.0.3", "@types/uuencode"); + }); test("should patch a package when it is already patched", async () => { const tempdir = tempDirWithFiles("lol", { "package.json": JSON.stringify({ From a83f6f0a9a0b3f0fc2af3a8e57f945c1850e697a Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:09:34 -0700 Subject: [PATCH 28/51] comment out unnecessary tests --- test/cli/install/bun-patch.test.ts | 54 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 6d88e416899120..37c3ca4296cc3a 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -56,33 +56,33 @@ describe("bun patch ", async () => { makeTest("react", "18.3.1"); makeTest("react-dom", "18.3.1"); makeTest("axios", "1.7.2"); - makeTest("tslib", "2.6.3"); - makeTest("chalk", "5.3.0"); - makeTest("next", "14.2.4"); - makeTest("express", "4.19.2"); - makeTest("inquirer", "9.2.23"); - makeTest("commander", "12.1.0"); - - // vercel/next.js - makeTest("webpack-sources", "3.2.3"); - - // vitejs/vite - makeTest("acorn", "8.11.3"); - makeTest("chokidar", "3.6.0"); - makeTest("http-proxy", "1.18.1"); - makeTest("sirv", "2.0.4"); - - // mermaid-js/mermaid - makeTest("cytoscape", "3.28.1"); - - // remix-run/react-router - makeTest("@changesets/get-dependents-graph", "1.3.6", "@changesets/get-dependents-graph"); - - // n8n-io/n8n - makeTest("typedi", "0.10.0"); - makeTest("@sentry/cli", "2.17.0", "@sentry/cli"); - makeTest("pkce-challenge", "3.0.0"); - makeTest("pyodide", "0.23.4"); + // makeTest("tslib", "2.6.3"); + // makeTest("chalk", "5.3.0"); + // makeTest("next", "14.2.4"); + // makeTest("express", "4.19.2"); + // makeTest("inquirer", "9.2.23"); + // makeTest("commander", "12.1.0"); + + // // vercel/next.js + // makeTest("webpack-sources", "3.2.3"); + + // // vitejs/vite + // makeTest("acorn", "8.11.3"); + // makeTest("chokidar", "3.6.0"); + // makeTest("http-proxy", "1.18.1"); + // makeTest("sirv", "2.0.4"); + + // // mermaid-js/mermaid + // makeTest("cytoscape", "3.28.1"); + + // // remix-run/react-router + // makeTest("@changesets/get-dependents-graph", "1.3.6", "@changesets/get-dependents-graph"); + + // // n8n-io/n8n + // makeTest("typedi", "0.10.0"); + // makeTest("@sentry/cli", "2.17.0", "@sentry/cli"); + // makeTest("pkce-challenge", "3.0.0"); + // makeTest("pyodide", "0.23.4"); makeTest("@types/express-serve-static-core", "4.17.43", "@types/express-serve-static-core"); makeTest("@types/ws", "8.5.4", "@types/ws"); makeTest("@types/uuencode", "0.0.3", "@types/uuencode"); From 9f00db62e750d7c25d16940feddd508138ca2053 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 17 Jun 2024 17:34:19 -0700 Subject: [PATCH 29/51] fix process double deref --- src/bun.js/api/bun/process.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bun.js/api/bun/process.zig b/src/bun.js/api/bun/process.zig index 699734afcd8c58..f5b52fc333ff48 100644 --- a/src/bun.js/api/bun/process.zig +++ b/src/bun.js/api/bun/process.zig @@ -1953,6 +1953,7 @@ pub const sync = struct { var this = SyncWindowsProcess.new(.{ .process = spawned.toProcess(undefined, true), }); + this.process.ref(); this.process.setExitHandler(this); defer this.destroy(); this.process.enableKeepingEventLoopAlive(); From 6a0975f771548810e7f4151841d50e680dcf1372 Mon Sep 17 00:00:00 2001 From: Zack Radisic Date: Tue, 18 Jun 2024 04:06:48 +0200 Subject: [PATCH 30/51] lol magic numbers! --- src/bun.js/api/bun/process.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bun.js/api/bun/process.zig b/src/bun.js/api/bun/process.zig index f5b52fc333ff48..3138bcf7a39dc9 100644 --- a/src/bun.js/api/bun/process.zig +++ b/src/bun.js/api/bun/process.zig @@ -2069,11 +2069,11 @@ pub const sync = struct { process.stderr orelse bun.invalid_fd, }; - if (process.memfds[0]) { + if (process.memfds[1]) { out_fds_to_wait_for[0] = bun.invalid_fd; } - if (process.memfds[1]) { + if (process.memfds[2]) { out_fds_to_wait_for[1] = bun.invalid_fd; } From a30dd112da9b7730fd4d447927e07afa77d3310f Mon Sep 17 00:00:00 2001 From: Zack Radisic Date: Tue, 18 Jun 2024 04:07:35 +0200 Subject: [PATCH 31/51] use package manager's temp directory --- src/install/install.zig | 4 ++-- src/install/patch_install.zig | 14 +++++--------- src/sys.zig | 1 + 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index ca58ecefc8a5f9..f02724aa039afb 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -11138,7 +11138,7 @@ pub const PackageManager = struct { // write the patch contents to temp file then rename var tmpname_buf: [1024]u8 = undefined; const tempfile_name = bun.span(try bun.fs.FileSystem.instance.tmpname("tmp", &tmpname_buf, bun.fastRandom())); - const tmpdir = try bun.fs.FileSystem.instance.tmpdir(); + const tmpdir = manager.getTemporaryDirectory(); const tmpfd = switch (bun.sys.openat( bun.toFD(tmpdir.fd), tempfile_name, @@ -11204,7 +11204,7 @@ pub const PackageManager = struct { path_in_patches_dir, ).asErr()) |e| { Output.prettyError( - "error: failed to renaming patch file to patches dir {}\n", + "error: failed renaming patch file to patches dir {}\n", .{e.toSystemError()}, ); Output.flush(); diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 490a68cd3aadfd..1d4e25da76bc3d 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -41,6 +41,7 @@ pub const BuntagHashBuf = [max_buntag_hash_buf_len]u8; pub const PatchTask = struct { manager: *PackageManager, + tempdir: std.fs.Dir, project_dir: []const u8, callback: union(enum) { calc_hash: CalcPatchHash, @@ -156,7 +157,7 @@ pub const PatchTask = struct { _ = manager; // autofix if (this.callback.apply.logger.errors > 0) { defer this.callback.apply.logger.deinit(); - // this.log.addErrorFmt(null, logger.Loc.Empty, bun.default_allocator, "failed to apply patch: {}", .{e}) catch unreachable; + Output.printErrorln("failed to apply patchfile ({s})", .{this.callback.apply.patchfilepath}); this.callback.apply.logger.printForLogLevel(Output.writer()) catch {}; } } @@ -298,14 +299,7 @@ pub const PatchTask = struct { // 2. Create temp dir to do all the modifications var tmpname_buf: [1024]u8 = undefined; const tempdir_name = bun.span(bun.fs.FileSystem.instance.tmpname("tmp", &tmpname_buf, bun.fastRandom()) catch bun.outOfMemory()); - const system_tmpdir = bun.fs.FileSystem.instance.tmpdir() catch |e| { - try log.addErrorFmtNoLoc( - this.manager.allocator, - "failed to creating temp dir: {s}", - .{@errorName(e)}, - ); - return; - }; + const system_tmpdir = this.tempdir; const pkg_name = this.callback.apply.pkgname; @@ -500,6 +494,7 @@ pub const PatchTask = struct { const patchfile_path = manager.allocator.dupeZ(u8, patchdep.path.slice(manager.lockfile.buffers.string_bytes.items)) catch bun.outOfMemory(); const pt = bun.new(PatchTask, .{ + .tempdir = manager.getTemporaryDirectory(), .callback = .{ .calc_hash = .{ .state = state, @@ -535,6 +530,7 @@ pub const PatchTask = struct { const patchfilepath = pkg_manager.allocator.dupe(u8, pkg_manager.lockfile.patched_dependencies.get(name_and_version_hash).?.path.slice(pkg_manager.lockfile.buffers.string_bytes.items)) catch bun.outOfMemory(); const pt = bun.new(PatchTask, .{ + .tempdir = pkg_manager.getTemporaryDirectory(), .callback = .{ .apply = .{ .pkg_id = pkg_id, diff --git a/src/sys.zig b/src/sys.zig index 527592d830b321..8873919f82d926 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -1695,6 +1695,7 @@ pub const RenameAt2Flags = packed struct { } }; +// NOTE: that this _does not_ handle moving across filesystems. For that, check if the return error is XDEV and then use `bun.C.moveFileZWithHandle` pub fn renameatConcurrently(from_dir_fd: bun.FileDescriptor, from: [:0]const u8, to_dir_fd: bun.FileDescriptor, to: [:0]const u8) Maybe(void) { var did_atomically_replace = false; From 48bc1a383334bfbd80553a563483e06f16acc122 Mon Sep 17 00:00:00 2001 From: Zack Radisic Date: Tue, 18 Jun 2024 04:09:24 +0200 Subject: [PATCH 32/51] =?UTF-8?q?=E2=9C=82=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cli.zig b/src/cli.zig index a5591a1558f54f..36a6d3e24b7574 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -1006,7 +1006,6 @@ pub const HelpCommand = struct { \\ link [\] Register or link a local npm package \\ unlink Unregister a local npm package \\ patch \ Prepare a package for patching - \\ patch-commit \ Install a modified package \\ pm \ Additional package management utilities \\ \\ build ./a.ts ./b.jsx Bundle TypeScript & JavaScript into a single file From 13b24175c329bff024b55666a3d72cb272595cb3 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:02:26 -0700 Subject: [PATCH 33/51] windows filepaths again man --- test/cli/install/bun-patch.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 37c3ca4296cc3a..45e74cfa3d087e 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -4,6 +4,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, expect, it, describe, test, import { join, sep } from "path"; const expectNoError = (o: ShellOutput) => expect(o.stderr.toString()).not.toContain("error"); +const platformPath = (path: string) => (process.platform === "win32" ? path.replaceAll("/", sep) : path); describe("bun patch ", async () => { // Tests to make sure that patching @@ -257,13 +258,13 @@ module.exports = function isOdd(i) { const { stderr, stdout } = await $`${bunExe()} patch ${patchArg}`.env(bunEnv).cwd(filedir); expect(stderr.toString()).not.toContain("error"); expect(stdout.toString()).toContain( - `To patch ${expected.patchName}, edit the following folder: + `To patch ${platformPath(expected.patchName)}, edit the following folder: - ${expected.patchPath} + ${platformPath(expected.patchPath)} Once you're done with your changes, run: - bun patch --commit '${expected.patchPath}'`, + bun patch --commit '${platformPath(expected.patchPath)}'`, ); } From 306147b37efacae8dbe605adfec95daf935a52fe Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:16:43 -0700 Subject: [PATCH 34/51] win paths --- src/install/install.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index f02724aa039afb..5b071bd5900ed1 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10222,6 +10222,8 @@ pub const PackageManager = struct { var iterator = Lockfile.Tree.Iterator.init(manager.lockfile); var resolution_buf: [1024]u8 = undefined; + var win_normalizer: if (bun.Environment.isWindows) bun.PathBuffer else struct {} = undefined; + const cache_dir: std.fs.Dir, const cache_dir_subpath: []const u8, const module_folder: []const u8, const pkg_name: []const u8 = switch (arg_kind) { .path => brk: { var lockfile = manager.lockfile; @@ -10314,10 +10316,12 @@ pub const PackageManager = struct { const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; + const buf = if (comptime bun.Environment.isWindows) bun.path.posixToPlatformInPlace(argument, win_normalizer[0..]) else argument; + break :brk .{ cache_dir, cache_dir_subpath, - argument, + buf, name, }; }, @@ -10361,11 +10365,13 @@ pub const PackageManager = struct { const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; - const module_folder = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); + const module_folder_ = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); + const buf = if (comptime bun.Environment.isWindows) bun.path.posixToPlatformInPlace(module_folder_, win_normalizer[0..]) else module_folder_; + break :brk .{ cache_dir, cache_dir_subpath, - module_folder, + buf, pkg_name, }; }, From 1358188cd9e99f629d8fa47649da3c1bca6a6a1e Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:19:14 -0700 Subject: [PATCH 35/51] win32 man --- src/install/install.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 5b071bd5900ed1..74263c0316231b 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10316,7 +10316,7 @@ pub const PackageManager = struct { const cache_dir = cache_result.cache_dir; const cache_dir_subpath = cache_result.cache_dir_subpath; - const buf = if (comptime bun.Environment.isWindows) bun.path.posixToPlatformInPlace(argument, win_normalizer[0..]) else argument; + const buf = if (comptime bun.Environment.isWindows) bun.path.pathToPosixBuf(u8, argument, win_normalizer[0..]) else argument; break :brk .{ cache_dir, @@ -10366,7 +10366,7 @@ pub const PackageManager = struct { const cache_dir_subpath = cache_result.cache_dir_subpath; const module_folder_ = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); - const buf = if (comptime bun.Environment.isWindows) bun.path.posixToPlatformInPlace(module_folder_, win_normalizer[0..]) else module_folder_; + const buf = if (comptime bun.Environment.isWindows) bun.path.pathToPosixBuf(u8, win_normalizer[0..]) else module_folder_; break :brk .{ cache_dir, From 1a429e05420f96e06283ab720bc943d73b908156 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:21:44 -0700 Subject: [PATCH 36/51] nice --- src/install/install.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install/install.zig b/src/install/install.zig index 74263c0316231b..749a9bf968bb7a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10366,7 +10366,7 @@ pub const PackageManager = struct { const cache_dir_subpath = cache_result.cache_dir_subpath; const module_folder_ = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); - const buf = if (comptime bun.Environment.isWindows) bun.path.pathToPosixBuf(u8, win_normalizer[0..]) else module_folder_; + const buf = if (comptime bun.Environment.isWindows) bun.path.pathToPosixBuf(u8, module_folder_, _normalizer[0..]) else module_folder_; break :brk .{ cache_dir, From e59b5774e822a46b6acbd3c585008e27163a7cdb Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:22:43 -0700 Subject: [PATCH 37/51] oops --- src/install/install.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install/install.zig b/src/install/install.zig index 749a9bf968bb7a..9b3b42efaadc81 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10366,7 +10366,7 @@ pub const PackageManager = struct { const cache_dir_subpath = cache_result.cache_dir_subpath; const module_folder_ = bun.path.join(&[_][]const u8{ folder.relative_path, name }, .auto); - const buf = if (comptime bun.Environment.isWindows) bun.path.pathToPosixBuf(u8, module_folder_, _normalizer[0..]) else module_folder_; + const buf = if (comptime bun.Environment.isWindows) bun.path.pathToPosixBuf(u8, module_folder_, win_normalizer[0..]) else module_folder_; break :brk .{ cache_dir, From 47e22738a396e973d0078009dd4bb11d385d7de1 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 17 Jun 2024 21:02:51 -0700 Subject: [PATCH 38/51] win32 thangs --- test/cli/install/bun-patch.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 45e74cfa3d087e..2bbd23090f8632 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -4,7 +4,8 @@ import { afterAll, afterEach, beforeAll, beforeEach, expect, it, describe, test, import { join, sep } from "path"; const expectNoError = (o: ShellOutput) => expect(o.stderr.toString()).not.toContain("error"); -const platformPath = (path: string) => (process.platform === "win32" ? path.replaceAll("/", sep) : path); +// const platformPath = (path: string) => (process.platform === "win32" ? path.replaceAll("/", sep) : path); +const platformPath = (path: string) => path describe("bun patch ", async () => { // Tests to make sure that patching @@ -236,6 +237,7 @@ module.exports = function isOdd(i) { extra?: (filedir: string) => Promise; }, ) { + expected.patchPath = platformPath(expected.patchPath); test(name, async () => { $.throws(true); @@ -306,7 +308,7 @@ Once you're done with your changes, run: const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); expect(stderr.toString()).not.toContain("error"); } - }); + }, 15 * 1000); ["is-even@1.0.0", "node_modules/is-even"].map(patchArg => makeTest("should patch a node_modules package", { From 85f17cdcb5a26d9023f90e79b627a2528425b091 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:19:54 -0700 Subject: [PATCH 39/51] resolve some comments --- src/install/install.zig | 294 +++++++++------------------------------- 1 file changed, 66 insertions(+), 228 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 9b3b42efaadc81..e0ca6fe9c97800 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -6784,12 +6784,12 @@ pub const PackageManager = struct { .workspaces = true, }, patch_features: union(enum) { - NOTHING: struct {}, + nothing: struct {}, patch: struct {}, commit: struct { patches_dir: string, }, - } = .{ .NOTHING = .{} }, + } = .{ .nothing = .{} }, // The idea here is: // 1. package has a platform-specific binary to install // 2. To prevent downloading & installing incompatible versions, they stick the "real" one in optionalDependencies @@ -7252,7 +7252,7 @@ pub const PackageManager = struct { if (!this.update.development) this.update.optional = cli.optional; switch (cli.patch) { - .NOTHING => {}, + .nothing => {}, .patch => { this.patch_features = .{ .patch = .{} }; }, @@ -7264,15 +7264,6 @@ pub const PackageManager = struct { }; }, } - // if (subcommand == .patch) { - // // TODO args - // } else if (subcommand == .@"patch-commit") { - // this.patch_features = .{ - // .commit = .{ - // .patches_dir = cli.@"patch-commit".patches_dir, - // }, - // }; - // } } else { this.log_level = if (default_disable_progress_bar) LogLevel.default_no_progress else LogLevel.default; PackageManager.verbose_install = false; @@ -9140,10 +9131,10 @@ pub const PackageManager = struct { concurrent_scripts: ?usize = null, - patch: PatchOpts = .{ .NOTHING = .{} }, + patch: PatchOpts = .{ .nothing = .{} }, const PatchOpts = union(enum) { - NOTHING: struct {}, + nothing: struct {}, patch: struct {}, commit: struct { patches_dir: []const u8 = "patches", @@ -9918,13 +9909,6 @@ pub const PackageManager = struct { }, else => { if (manager.options.patch_features == .commit) { - // _ = manager.lockfile.loadFromDisk( - // manager, - // manager.allocator, - // manager.log, - // manager.options.lockfile_path, - // true, - // ); var pathbuf: bun.PathBuffer = undefined; if (try manager.doPatchCommit(&pathbuf, log_level)) |stuff| { try PackageJSONEditor.editPatchedDependencies( @@ -10297,9 +10281,10 @@ pub const PackageManager = struct { }; const existing_patchfile_hash = existing_patchfile_hash: { - var sfb = std.heap.stackFallback(1024, manager.allocator); - const name_and_version = std.fmt.allocPrint(sfb.get(), "{s}@{}", .{ name, actual_package.resolution.fmt(strbuf, .posix) }) catch unreachable; - defer sfb.get().free(name_and_version); + var __sfb = std.heap.stackFallback(1024, manager.allocator); + const sfballoc = __sfb.get(); + const name_and_version = std.fmt.allocPrint(sfballoc, "{s}@{}", .{ name, actual_package.resolution.fmt(strbuf, .posix) }) catch unreachable; + defer sfballoc.free(name_and_version); const name_and_version_hash = String.Builder.stringHash(name_and_version); if (lockfile.patched_dependencies.get(name_and_version_hash)) |patched_dep| { if (patched_dep.patchfileHash()) |hash| break :existing_patchfile_hash hash; @@ -10345,9 +10330,10 @@ pub const PackageManager = struct { const pkg_name = pkg.name.slice(strbuf); const existing_patchfile_hash = existing_patchfile_hash: { - var sfb = std.heap.stackFallback(1024, manager.allocator); - const name_and_version = std.fmt.allocPrint(sfb.get(), "{s}@{}", .{ name, pkg.resolution.fmt(strbuf, .posix) }) catch unreachable; - defer sfb.get().free(name_and_version); + var __sfb = std.heap.stackFallback(1024, manager.allocator); + const sfballoc = __sfb.get(); + const name_and_version = std.fmt.allocPrint(sfballoc, "{s}@{}", .{ name, pkg.resolution.fmt(strbuf, .posix) }) catch unreachable; + defer sfballoc.free(name_and_version); const name_and_version_hash = String.Builder.stringHash(name_and_version); if (manager.lockfile.patched_dependencies.get(name_and_version_hash)) |patched_dep| { if (patched_dep.patchfileHash()) |hash| break :existing_patchfile_hash hash; @@ -10415,6 +10401,10 @@ pub const PackageManager = struct { pub fn copy( destination_dir_: std.fs.Dir, walker: *Walker, + in_dir: if (bun.Environment.isWindows) []const u8 else void, + out_dir: if (bun.Environment.isWindows) []const u8 else void, + buf1: if (bun.Environment.isWindows) []u8 else void, + buf2: if (bun.Environment.isWindows) []u8 else void, ) !u32 { var real_file_count: u32 = 0; @@ -10459,34 +10449,22 @@ pub const PackageManager = struct { Output.prettyError("error: opening node_modules file {s} {s} ()", .{ @errorName(e), entrypath }); Global.crash(); }; - defer in_file.close(); + var in_file_close = true; + defer if (in_file_close) in_file.close(); const mode = in_file.mode() catch @panic("OH NO"); var outfile = createFile(destination_dir_, entrypath, .{ .mode = mode }) catch @panic("OH NO"); - defer outfile.close(); - - var buf1: bun.windows.WPathBuffer = undefined; - var buf2: bun.windows.WPathBuffer = undefined; - - const infile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(in_file.handle, &buf1, buf1.len, 0); - if (infile_path_len == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - } - buf1[infile_path_len] = 0; + var out_file_close = true; + defer if (out_file_close) outfile.close(); - const outfile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(outfile.handle, &buf2, buf2.len, 0); - if (outfile_path_len == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - Output.prettyError("error: copying file {}", .{err}); - Global.crash(); - } - buf2[outfile_path_len] = 0; + const infile_path = bun.path.joinZBuf(buf1, &[_][]const u8{ in_dir, entrypath }, .auto); + const outfile_path = bun.path.joinZBuf(buf2, &[_][]const u8{ out_dir, entrypath }, .auto); - bun.copyFileWithState(buf1[0..infile_path_len :0], buf2[0..outfile_path_len :0], ©_file_state) catch |err| { + in_file_close = false; + out_file_close = false; + in_file.close(); + outfile.close(); + bun.copyFileWithState(infile_path, outfile_path, ©_file_state) catch |err| { Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); Global.crash(); }; @@ -10511,23 +10489,6 @@ pub const PackageManager = struct { var outfile = try createFile(destination_dir_, entry.path, .{}); defer outfile.close(); - // debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); - // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { - // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { - // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; - // } - // break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { - // if (do_progress) { - // progress_.root.end(); - // progress_.refresh(); - // } - - // Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - // Global.crash(); - // }; - // }; - // defer outfile.close(); - if (comptime Environment.isPosix) { const stat = in_file.stat() catch continue; _ = C.fchmod(outfile.handle, @intCast(stat.mode)); @@ -10547,173 +10508,50 @@ pub const PackageManager = struct { defer pkg_in_cache_dir.close(); var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); defer walker.deinit(); + + var buf1: if (bun.Environment.isWindows) bun.PathBuffer else void = undefined; + var buf2: if (bun.Environment.isWindows) bun.PathBuffer else void = undefined; + var in_dir: if (bun.Environment.isWindows) []const u8 else void = undefined; + var out_dir: if (bun.Environment.isWindows) []const u8 else void = undefined; + + if (comptime bun.Environment.isWindows) { + const inlen = bun.windows.kernel32.GetFinalPathNameByHandleW(pkg_in_cache_dir.fd, &buf1, buf1.len, 0); + if (inlen == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + in_dir = buf1[0..inlen]; + const outlen = bun.windows.kernel32.GetFinalPathNameByHandleW(node_modules_folder.fd, &buf2, buf2.len, 0); + if (outlen == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + Output.prettyError("error: copying file {}", .{err}); + Global.crash(); + } + out_dir = buf2[0..outlen]; + _ = try FileCopier.copy( + node_modules_folder, + &walker, + in_dir, + out_dir, + &buf1, + &buf2, + ); + return; + } + _ = try FileCopier.copy( node_modules_folder, &walker, + {}, + {}, + {}, + {}, ); } - // fn overwriteNodeModulesFolder( - // manager: *PackageManager, - // cache_dir: std.fs.Dir, - // cache_dir_subpath: []const u8, - // node_modules_folder_path: []const u8, - // ) !void { - // var node_modules_folder = try std.fs.cwd().openDir(node_modules_folder_path, .{ .iterate = true }); - // defer node_modules_folder.close(); - - // const IGNORED_PATHS: []const bun.OSPathSlice = &[_][]const bun.OSPathChar{ - // bun.OSPathLiteral("node_modules"), - // bun.OSPathLiteral(".git"), - // bun.OSPathLiteral("CMakeFiles"), - // }; - - // const FileCopier = struct { - // pub fn copy( - // destination_dir_: std.fs.Dir, - // walker: *Walker, - // ) !u32 { - // var real_file_count: u32 = 0; - - // var copy_file_state: bun.CopyFileState = .{}; - // var pathbuf: bun.PathBuffer = undefined; - // var pathbuf2: bun.PathBuffer = undefined; - // // _ = pathbuf; // autofix - - // while (try walker.next()) |entry| { - // if (entry.kind != .file) continue; - // real_file_count += 1; - // const openFile = std.fs.Dir.openFile; - // const createFile = std.fs.Dir.createFile; - - // // - rename node_modules/$PKG/$FILE -> node_modules/$PKG/$TMPNAME - // // - create node_modules/$PKG/$FILE - // // - copy: cache/$PKG/$FILE -> node_modules/$PKG/$FILE - // // - unlink: $TMPDIR/$FILE - // if (comptime bun.Environment.isWindows) { - // var tmpbuf: [1024]u8 = undefined; - // const basename = bun.strings.fromWPath(pathbuf2[0..], entry.basename); - // const tmpname = bun.span(bun.fs.FileSystem.instance.tmpname(basename, tmpbuf[0..], bun.fastRandom()) catch |e| { - // Output.prettyError("error: copying file {s}", .{@errorName(e)}); - // Global.crash(); - // }); - - // const entrypath = bun.strings.fromWPath(pathbuf[0..], entry.path); - // pathbuf[entrypath.len] = 0; - // const entrypathZ = pathbuf[0..entrypath.len :0]; - - // if (bun.sys.renameatConcurrently( - // bun.toFD(destination_dir_.fd), - // entrypathZ, - // bun.toFD(destination_dir_.fd), - // tmpname, - // ).asErr()) |e| { - // Output.prettyError("error: copying file {}", .{e}); - // Global.crash(); - // } - - // var in_file = try openFile(entry.dir, entrypath, .{ .mode = .read_only }); - // defer in_file.close(); - - // if (bun.sys.unlinkat( - // bun.toFD(destination_dir_.fd), - // entrypath, - // ).asErr()) |e| { - // Output.prettyError("error: copying file {}", .{e.withPath(entrypath)}); - // Global.crash(); - // } - - // const mode = try in_file.mode(); - // var outfile = try createFile(destination_dir_, entrypath, .{ .mode = mode }); - // defer outfile.close(); - - // var buf1: bun.windows.WPathBuffer = undefined; - // var buf2: bun.windows.WPathBuffer = undefined; - - // const infile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(in_file.handle, &buf1, buf1.len, 0); - // if (infile_path_len == 0) { - // const e = bun.windows.Win32Error.get(); - // const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - // Output.prettyError("error: copying in path {}", .{err}); - // Global.crash(); - // } - // buf1[infile_path_len] = 0; - - // const outfile_path_len = bun.windows.kernel32.GetFinalPathNameByHandleW(outfile.handle, &buf2, buf2.len, 0); - // if (outfile_path_len == 0) { - // const e = bun.windows.Win32Error.get(); - // const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; - // Output.prettyError("error: getting out path {}", .{err}); - // Global.crash(); - // } - // buf2[outfile_path_len] = 0; - // bun.copyFileWithState(buf1[0..infile_path_len :0], buf2[0..outfile_path_len :0], ©_file_state) catch |err| { - // Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - // Global.crash(); - // }; - - // continue; - // } - - // var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); - // defer in_file.close(); - - // @memcpy(pathbuf[0..entry.path.len], entry.path); - // pathbuf[entry.path.len] = 0; - - // if (bun.sys.unlinkat( - // bun.toFD(destination_dir_.fd), - // pathbuf[0..entry.path.len :0], - // ).asErr()) |e| { - // Output.prettyError("error: copying file {}", .{e.withPath(entry.path)}); - // Global.crash(); - // } - - // var outfile = try createFile(destination_dir_, entry.path, .{}); - // defer outfile.close(); - - // // debug("createFile {} {s}\n", .{ destination_dir_.fd, entry.path }); - // // var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { - // // if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { - // // bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; - // // } - // // break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { - // // if (do_progress) { - // // progress_.root.end(); - // // progress_.refresh(); - // // } - - // // Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - // // Global.crash(); - // // }; - // // }; - // // defer outfile.close(); - - // if (comptime Environment.isPosix) { - // const stat = in_file.stat() catch continue; - // _ = C.fchmod(outfile.handle, @intCast(stat.mode)); - // } - - // bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { - // Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - // Global.crash(); - // }; - // } - - // return real_file_count; - // } - // }; - - // var pkg_in_cache_dir = try cache_dir.openDir(cache_dir_subpath, .{ .iterate = true }); - // defer pkg_in_cache_dir.close(); - // var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); - // defer walker.deinit(); - // _ = try FileCopier.copy( - // node_modules_folder, - // &walker, - // ); - // } - const PatchCommitResult = struct { patch_key: []const u8, patchfile_path: []const u8, From ff234d4db68a3205006161f8fb23e8d53ed4b8fb Mon Sep 17 00:00:00 2001 From: zackradisic Date: Tue, 18 Jun 2024 05:21:04 +0000 Subject: [PATCH 40/51] Apply formatting changes --- test/cli/install/bun-patch.test.ts | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index 2bbd23090f8632..a473fbb10b8b6a 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -5,7 +5,7 @@ import { join, sep } from "path"; const expectNoError = (o: ShellOutput) => expect(o.stderr.toString()).not.toContain("error"); // const platformPath = (path: string) => (process.platform === "win32" ? path.replaceAll("/", sep) : path); -const platformPath = (path: string) => path +const platformPath = (path: string) => path; describe("bun patch ", async () => { // Tests to make sure that patching @@ -282,33 +282,37 @@ Once you're done with your changes, run: }); } - test("overwriting module with multiple levels of directories", async () => { - const filedir = tempDirWithFiles("patch1", { - "package.json": JSON.stringify({ - "name": "bun-patch-test", - "module": "index.ts", - "type": "module", - "dependencies": { lodash: "4.17.21" }, - }), - "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, - }); + test( + "overwriting module with multiple levels of directories", + async () => { + const filedir = tempDirWithFiles("patch1", { + "package.json": JSON.stringify({ + "name": "bun-patch-test", + "module": "index.ts", + "type": "module", + "dependencies": { lodash: "4.17.21" }, + }), + "index.ts": /* ts */ `import isEven from 'is-even'; console.log(isEven())`, + }); - { - const { stderr } = await $`${bunExe()} i`.env(bunEnv).cwd(filedir); - expect(stderr.toString()).not.toContain("error"); - } + { + const { stderr } = await $`${bunExe()} i`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } - { - const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); - expect(stderr.toString()).not.toContain("error"); - } + { + const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } - // run it again to make sure we didn't f something up - { - const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); - expect(stderr.toString()).not.toContain("error"); - } - }, 15 * 1000); + // run it again to make sure we didn't f something up + { + const { stderr, stdout } = await $`${bunExe()} patch lodash`.env(bunEnv).cwd(filedir); + expect(stderr.toString()).not.toContain("error"); + } + }, + 15 * 1000, + ); ["is-even@1.0.0", "node_modules/is-even"].map(patchArg => makeTest("should patch a node_modules package", { From cc40bf6146352d9b108c5cba841ab42c97012a98 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:31:42 -0700 Subject: [PATCH 41/51] fix windows build --- src/install/install.zig | 20 ++++++++++---------- src/resolver/resolve_path.zig | 8 ++++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index e0ca6fe9c97800..be523364d65799 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10401,10 +10401,10 @@ pub const PackageManager = struct { pub fn copy( destination_dir_: std.fs.Dir, walker: *Walker, - in_dir: if (bun.Environment.isWindows) []const u8 else void, - out_dir: if (bun.Environment.isWindows) []const u8 else void, - buf1: if (bun.Environment.isWindows) []u8 else void, - buf2: if (bun.Environment.isWindows) []u8 else void, + in_dir: if (bun.Environment.isWindows) []const u16 else void, + out_dir: if (bun.Environment.isWindows) []const u16 else void, + buf1: if (bun.Environment.isWindows) []u16 else void, + buf2: if (bun.Environment.isWindows) []u16 else void, ) !u32 { var real_file_count: u32 = 0; @@ -10457,8 +10457,8 @@ pub const PackageManager = struct { var out_file_close = true; defer if (out_file_close) outfile.close(); - const infile_path = bun.path.joinZBuf(buf1, &[_][]const u8{ in_dir, entrypath }, .auto); - const outfile_path = bun.path.joinZBuf(buf2, &[_][]const u8{ out_dir, entrypath }, .auto); + const infile_path = bun.path.joinStringBufWZ(buf1, &[_][]const u16{ in_dir, entry.path }, .auto); + const outfile_path = bun.path.joinStringBufWZ(buf2, &[_][]const u16{ out_dir, entry.path }, .auto); in_file_close = false; out_file_close = false; @@ -10509,10 +10509,10 @@ pub const PackageManager = struct { var walker = Walker.walk(pkg_in_cache_dir, manager.allocator, &.{}, IGNORED_PATHS) catch bun.outOfMemory(); defer walker.deinit(); - var buf1: if (bun.Environment.isWindows) bun.PathBuffer else void = undefined; - var buf2: if (bun.Environment.isWindows) bun.PathBuffer else void = undefined; - var in_dir: if (bun.Environment.isWindows) []const u8 else void = undefined; - var out_dir: if (bun.Environment.isWindows) []const u8 else void = undefined; + var buf1: if (bun.Environment.isWindows) bun.WPathBuffer else void = undefined; + var buf2: if (bun.Environment.isWindows) bun.WPathBuffer else void = undefined; + var in_dir: if (bun.Environment.isWindows) []const u16 else void = undefined; + var out_dir: if (bun.Environment.isWindows) []const u16 else void = undefined; if (comptime bun.Environment.isWindows) { const inlen = bun.windows.kernel32.GetFinalPathNameByHandleW(pkg_in_cache_dir.fd, &buf1, buf1.len, 0); diff --git a/src/resolver/resolve_path.zig b/src/resolver/resolve_path.zig index 53af0d083e6c9a..f2af9c97c7fb92 100644 --- a/src/resolver/resolve_path.zig +++ b/src/resolver/resolve_path.zig @@ -1258,6 +1258,14 @@ pub fn joinStringBufW(buf: []u16, parts: anytype, comptime _platform: Platform) return joinStringBufT(u16, buf, parts, _platform); } +pub fn joinStringBufWZ(buf: []u16, parts: anytype, comptime _platform: Platform) [:0]const u16 { + const joined = joinStringBufT(u16, buf[0 .. buf.len - 1], parts, _platform); + assert(bun.isSliceInBufferT(u16, joined, buf)); + const start_offset = @intFromPtr(joined.ptr) / 2 - @intFromPtr(buf.ptr) / 2; + buf[joined.len + start_offset] = 0; + return buf[start_offset..][0..joined.len :0]; +} + pub fn joinStringBufT(comptime T: type, buf: []T, parts: anytype, comptime _platform: Platform) []const T { const platform = comptime _platform.resolve(); From ad048d0a0b565643aaaec3762203a34300e69cd0 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:40:47 -0700 Subject: [PATCH 42/51] remove some panics --- src/install/install.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index be523364d65799..79bcd98d7fa88a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -3274,7 +3274,7 @@ pub const PackageManager = struct { // 3. apply patch to temp dir // 4. rename temp dir to `folder_path` if (patch_hash != null) { - const non_patched_path_ = folder_path[0 .. std.mem.indexOf(u8, folder_path, "_patch_hash=") orelse @panic("todo this is bad")]; + const non_patched_path_ = folder_path[0 .. std.mem.indexOf(u8, folder_path, "_patch_hash=") orelse @panic("Expected folder path to contain `patch_hash=`, this is a bug in Bun. Please file a GitHub issue.")]; const non_patched_path = manager.lockfile.allocator.dupeZ(u8, non_patched_path_) catch bun.outOfMemory(); defer manager.lockfile.allocator.free(non_patched_path); if (manager.isFolderInCache(non_patched_path)) { @@ -10452,8 +10452,14 @@ pub const PackageManager = struct { var in_file_close = true; defer if (in_file_close) in_file.close(); - const mode = in_file.mode() catch @panic("OH NO"); - var outfile = createFile(destination_dir_, entrypath, .{ .mode = mode }) catch @panic("OH NO"); + const mode = in_file.mode() catch |e| { + Output.prettyError("error: failed to get file mode {s} ", .{@errorName(e)}); + Global.crash(); + }; + var outfile = createFile(destination_dir_, entrypath, .{ .mode = mode }) catch |e| { + Output.prettyError("error: failed to create file {s} ({s})", .{ entrypath, @errorName(e) }); + Global.crash(); + }; var out_file_close = true; defer if (out_file_close) outfile.close(); From 2028d205c8a464f7d119078b78b656af4782b1d9 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:52:18 -0700 Subject: [PATCH 43/51] small things --- src/install/install.zig | 6 +- src/patch.zig | 264 ---------------------------------------- 2 files changed, 3 insertions(+), 267 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 79bcd98d7fa88a..5e209a1136f522 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10113,7 +10113,7 @@ pub const PackageManager = struct { return null; } - fn pkg_dep_id_for_name_and_version( + fn pkgDepIdForNameAndVersion( lockfile: *Lockfile, pkg_maybe_version_to_patch: []const u8, name: []const u8, @@ -10313,7 +10313,7 @@ pub const PackageManager = struct { .name_and_version => brk: { const pkg_maybe_version_to_patch = argument; const name, const version = Dependency.splitNameAndVersion(pkg_maybe_version_to_patch); - const result = pkg_dep_id_for_name_and_version(manager.lockfile, pkg_maybe_version_to_patch, name, version); + const result = pkgDepIdForNameAndVersion(manager.lockfile, pkg_maybe_version_to_patch, name, version); const pkg_id = result[0]; const dependency_id = result[1]; @@ -10717,7 +10717,7 @@ pub const PackageManager = struct { }, .name_and_version => brk: { const name, const version = Dependency.splitNameAndVersion(argument); - const result = pkg_dep_id_for_name_and_version(lockfile, argument, name, version); + const result = pkgDepIdForNameAndVersion(lockfile, argument, name, version); const pkg_id: PackageID = result[0]; const dependency_id: DependencyID = result[1]; const node_modules = (try nodeModulesFolderForDependencyID( diff --git a/src/patch.zig b/src/patch.zig index 9ca3c53b0cdf7b..f4d039ec615a37 100644 --- a/src/patch.zig +++ b/src/patch.zig @@ -1320,270 +1320,6 @@ pub fn diffPostProcess(result: *bun.spawn.sync.Result, old_folder: []const u8, n return .{ .result = stdout }; } -pub const Subproc = struct { - const Process = bun.spawn.Process; - const PackageManager = bun.install.PackageManager; - const uv = bun.windows.libuv; - pub const OutputReader = bun.io.BufferedReader; - - manager: *PackageManager, - process: ?*Process = null, - stdout: OutputReader = OutputReader.init(@This()), - stderr: OutputReader = OutputReader.init(@This()), - envp: [:null]?[*:0]const u8, - argv: [:null]?[*:0]const u8, - - is_done: bool = false, - - cwd: [:0]const u8, - old_folder: [:0]const u8, - new_folder: [:0]const u8, - - const ARGV = &[_][:0]const u8{ - "git", - "-c", - "core.safecrlf=false", - "diff", - "--src-prefix=a/", - "--dst-prefix=b/", - "--ignore-cr-at-eol", - "--irreversible-delete", - "--full-index", - "--no-index", - }; - - pub usingnamespace bun.New(@This()); - - pub fn resetPolls(this: *Subproc) void { - if (this.process) |process| { - this.process = null; - process.close(); - process.deref(); - } - - this.stdout.deinit(); - this.stderr.deinit(); - this.stdout = OutputReader.init(@This()); - this.stderr = OutputReader.init(@This()); - } - - pub fn deinit(this: *Subproc) void { - this.resetPolls(); - - this.stdout.deinit(); - this.stderr.deinit(); - - bun.default_allocator.free(this.old_folder); - bun.default_allocator.free(this.new_folder); - - bun.default_allocator.free(this.envp); - bun.default_allocator.free(this.argv); - - this.destroy(); - } - - pub fn isDone(this: *Subproc) bool { - return this.is_done; - } - - pub fn diffPostProcess(this: *Subproc) !bun.JSC.Node.Maybe(std.ArrayList(u8), std.ArrayList(u8)) { - var stdout = this.stdout.takeBuffer(); - var stderr = this.stdout.takeBuffer(); - - var deinit_stdout = true; - var deinit_stderr = true; - defer { - if (deinit_stdout) stdout.deinit(); - if (deinit_stderr) stderr.deinit(); - } - - if (stderr.items.len > 0) { - deinit_stderr = false; - return .{ .err = stderr }; - } - - debug("Before postprocess: {s}\n", .{stdout.items}); - try gitDiffPostprocess(&stdout, this.old_folder, this.new_folder); - deinit_stdout = false; - return .{ .result = stdout }; - } - - pub fn init( - manager: *PackageManager, - old_folder_: []const u8, - new_folder_: []const u8, - cwd: [:0]const u8, - git: [:0]const u8, - ) *Subproc { - const paths = gitDiffPreprocessPaths(bun.default_allocator, old_folder_, new_folder_, true); - - const giiit = bun.default_allocator.dupeZ(u8, git) catch bun.outOfMemory(); - const argv: [:null]?[*:0]const u8 = brk: { - const argv_buf = bun.default_allocator.allocSentinel(?[*:0]const u8, ARGV.len + 2, null) catch bun.outOfMemory(); - argv_buf[0] = giiit.ptr; - for (1..ARGV.len) |i| { - argv_buf[i] = ARGV[i].ptr; - } - argv_buf[ARGV.len] = paths[0]; - argv_buf[ARGV.len + 1] = paths[1]; - break :brk argv_buf; - }; - - const envp: [:null]?[*:0]const u8 = brk: { - const env_arr = &[_][:0]const u8{ - "GIT_CONFIG_NOSYSTEM", - "HOME", - "XDG_CONFIG_HOME", - "USERPROFILE", - }; - const PATH = bun.getenvZ("PATH"); - const envp_buf = bun.default_allocator.allocSentinel(?[*:0]const u8, env_arr.len + @as(usize, if (PATH != null) 1 else 0), null) catch bun.outOfMemory(); - for (0..env_arr.len) |i| { - envp_buf[i] = env_arr[i].ptr; - } - if (PATH) |p| { - envp_buf[envp_buf.len - 1] = @ptrCast(p.ptr); - } - break :brk envp_buf; - }; - - return Subproc.new(Subproc{ - .old_folder = paths[0], - .new_folder = paths[1], - .cwd = cwd, - .argv = argv, - .envp = envp, - .manager = manager, - }); - } - - pub fn loop(this: *const Subproc) *bun.uws.Loop { - return this.manager.event_loop.loop(); - } - - pub fn eventLoop(this: *const Subproc) *JSC.AnyEventLoop { - return &this.manager.event_loop; - } - - pub fn onReaderError(this: *Subproc, err: bun.sys.Error) void { - Output.prettyErrorln("error: Failed to git diff due to error {s}", .{ - @tagName(err.getErrno()), - }); - Output.flush(); - this.maybeFinished(); - } - - pub fn onReaderDone(this: *Subproc) void { - this.maybeFinished(); - } - - fn maybeFinished(this: *Subproc) void { - const process = this.process orelse return; - - this.handleExit(process.status); - } - - fn handleExit(this: *Subproc, status: bun.spawn.Status) void { - defer this.is_done = true; - - switch (status) { - // we can't rely on the exit code because git diff returns 1 even when - // it succeeds - .exited => {}, - .signaled => |signal| { - Output.prettyError( - "error: git diff terminated with{}\n", - .{bun.SignalCode.from(signal).fmt(Output.enable_ansi_colors_stderr)}, - ); - Output.flush(); - Global.crash(); - }, - .err => |err| { - Output.prettyError( - "error: failed to make diff {}\n", - .{err}, - ); - Output.flush(); - Global.crash(); - }, - else => {}, - } - } - - pub fn spawn(this: *Subproc) !void { - this.stdout.setParent(this); - this.stderr.setParent(this); - - if (bun.Environment.isWindows) { - this.stdout.source = .{ .pipe = bun.default_allocator.create(uv.Pipe) catch bun.outOfMemory() }; - this.stderr.source = .{ .pipe = bun.default_allocator.create(uv.Pipe) catch bun.outOfMemory() }; - } - - const spawn_options = bun.spawn.SpawnOptions{ - .stdin = .ignore, - .stdout = if (bun.Environment.isPosix) .buffer else .{ .buffer = this.stdout.source.?.pipe }, - .stderr = if (bun.Environment.isPosix) .buffer else .{ .buffer = this.stdout.source.?.pipe }, - - .cwd = this.cwd, - - .windows = if (bun.Environment.isWindows) - .{ .loop = JSC.EventLoopHandle.init(&this.manager.event_loop) } - else {}, - - .stream = false, - }; - - var spawned = try (try bun.spawn.spawnProcess(&spawn_options, this.argv, this.envp)).unwrap(); - - if (comptime bun.Environment.isPosix) { - if (spawned.stdout) |stdout| { - if (!spawned.memfds[1]) { - this.stdout.setParent(this); - _ = bun.sys.setNonblocking(stdout); - try this.stdout.start(stdout, true).unwrap(); - } else { - this.stdout.setParent(this); - this.stdout.startMemfd(stdout); - } - } - if (spawned.stderr) |stderr| { - if (!spawned.memfds[2]) { - this.stderr.setParent(this); - _ = bun.sys.setNonblocking(stderr); - try this.stderr.start(stderr, true).unwrap(); - } else { - this.stderr.setParent(this); - this.stderr.startMemfd(stderr); - } - } - } else if (comptime bun.Environment.isWindows) { - if (spawned.stdout == .buffer) { - this.stdout.parent = this; - try this.stdout.startWithCurrentPipe().unwrap(); - } - if (spawned.stderr == .buffer) { - this.stderr.parent = this; - try this.stderr.startWithCurrentPipe().unwrap(); - } - } - - const event_loop = &this.manager.event_loop; - var process = spawned.toProcess( - event_loop, - false, - ); - - this.process = process; - switch (process.watchOrReap()) { - .err => |err| { - if (!process.hasExited()) - process.onExit(.{ .err = err }, &std.mem.zeroes(bun.spawn.Rusage)); - }, - .result => {}, - } - } -}; - pub fn gitDiffPreprocessPaths( allocator: std.mem.Allocator, old_folder_: []const u8, From fb76ddbf93ac4790e5788c57e09906cbb8132cf8 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 22:59:19 -0700 Subject: [PATCH 44/51] remove dead commented out code --- src/install/install.zig | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 5e209a1136f522..6b7c4d572f697a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -6049,12 +6049,6 @@ pub const PackageManager = struct { if (ptask.callback.apply.task_id) |task_id| { _ = task_id; // autofix - // const name = manager.lockfile.packages.items(.name)[ptask.callback.apply.pkg_id].slice(manager.lockfile.buffers.string_bytes.items); - // if (!callbacks.onPatch(extract_ctx, name, task_id, log_level)) { - // if (comptime Environment.allow_assert) { - // Output.panic("Ran callback to install enqueued packages, but there was no task associated with it.", .{}); - // } - // } } else if (ExtractCompletionContext == *PackageInstaller) { if (ptask.callback.apply.install_context) |*ctx| { var installer: *PackageInstaller = extract_ctx; From ca75bd2dbfaed2c8953aa441361eb88a7dd73be5 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:09:25 -0700 Subject: [PATCH 45/51] bunch of fixes to resolve comments --- src/install/install.zig | 66 +++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 6b7c4d572f697a..b0b66735fa39cd 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10276,9 +10276,9 @@ pub const PackageManager = struct { const existing_patchfile_hash = existing_patchfile_hash: { var __sfb = std.heap.stackFallback(1024, manager.allocator); - const sfballoc = __sfb.get(); - const name_and_version = std.fmt.allocPrint(sfballoc, "{s}@{}", .{ name, actual_package.resolution.fmt(strbuf, .posix) }) catch unreachable; - defer sfballoc.free(name_and_version); + const allocator = __sfb.get(); + const name_and_version = std.fmt.allocPrint(allocator, "{s}@{}", .{ name, actual_package.resolution.fmt(strbuf, .posix) }) catch unreachable; + defer allocator.free(name_and_version); const name_and_version_hash = String.Builder.stringHash(name_and_version); if (lockfile.patched_dependencies.get(name_and_version_hash)) |patched_dep| { if (patched_dep.patchfileHash()) |hash| break :existing_patchfile_hash hash; @@ -10366,7 +10366,6 @@ pub const PackageManager = struct { "error: error overwriting folder in node_modules: {s}\n", .{@errorName(e)}, ); - Output.flush(); Global.crash(); }; @@ -10470,34 +10469,32 @@ pub const PackageManager = struct { }; continue; - } + } else if (comptime Environment.isPosix) { + var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); + defer in_file.close(); - var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); - defer in_file.close(); + @memcpy(pathbuf[0..entry.path.len], entry.path); + pathbuf[entry.path.len] = 0; - @memcpy(pathbuf[0..entry.path.len], entry.path); - pathbuf[entry.path.len] = 0; - - if (bun.sys.unlinkat( - bun.toFD(destination_dir_.fd), - pathbuf[0..entry.path.len :0], - ).asErr()) |e| { - Output.prettyError("error: copying file {}", .{e.withPath(entry.path)}); - Global.crash(); - } + if (bun.sys.unlinkat( + bun.toFD(destination_dir_.fd), + pathbuf[0..entry.path.len :0], + ).asErr()) |e| { + Output.prettyError("error: copying file {}", .{e.withPath(entry.path)}); + Global.crash(); + } - var outfile = try createFile(destination_dir_, entry.path, .{}); - defer outfile.close(); + var outfile = try createFile(destination_dir_, entry.path, .{}); + defer outfile.close(); - if (comptime Environment.isPosix) { const stat = in_file.stat() catch continue; _ = C.fchmod(outfile.handle, @intCast(stat.mode)); - } - bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { - Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; + bun.copyFileWithState(in_file.handle, outfile.handle, ©_file_state) catch |err| { + Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; + } } return real_file_count; @@ -10539,17 +10536,16 @@ pub const PackageManager = struct { &buf1, &buf2, ); - return; + } else if (Environment.isPosix) { + _ = try FileCopier.copy( + node_modules_folder, + &walker, + {}, + {}, + {}, + {}, + ); } - - _ = try FileCopier.copy( - node_modules_folder, - &walker, - {}, - {}, - {}, - {}, - ); } const PatchCommitResult = struct { From f43b1be745e4353b396e301fa43e5a659db01841 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:13:22 -0700 Subject: [PATCH 46/51] resolve comment --- src/install/patch_install.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 1d4e25da76bc3d..8ba1d94f9b4b60 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -157,7 +157,7 @@ pub const PatchTask = struct { _ = manager; // autofix if (this.callback.apply.logger.errors > 0) { defer this.callback.apply.logger.deinit(); - Output.printErrorln("failed to apply patchfile ({s})", .{this.callback.apply.patchfilepath}); + Output.errGeneric("failed to apply patchfile ({s})", .{this.callback.apply.patchfilepath}); this.callback.apply.logger.printForLogLevel(Output.writer()) catch {}; } } From 0665fc2da122e2388cf18dace1bbbfc6034dbd4e Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:19:10 -0700 Subject: [PATCH 47/51] remove Output.flush() before Global.crash() --- src/install/install.zig | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index b0b66735fa39cd..9b52c83a619520 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10136,7 +10136,6 @@ pub const PackageManager = struct { const first_match = maybe_first_match orelse { Output.prettyErrorln("\nerror: package {s} not found", .{pkg_maybe_version_to_patch}); - Output.flush(); Global.crash(); return; }; @@ -10158,7 +10157,6 @@ pub const PackageManager = struct { Output.prettyError(" {s}@{}\n", .{ pkg.name.slice(strbuf), pkg.resolution.fmt(strbuf, .posix) }); } else break; } - Output.flush(); Global.crash(); return; } @@ -10215,7 +10213,6 @@ pub const PackageManager = struct { "error: failed to read package.json: {}\n", .{e.withPath(package_json_path).toSystemError()}, ); - Output.flush(); Global.crash(); }, } @@ -10241,7 +10238,6 @@ pub const PackageManager = struct { "error: invalid package.json, missing or invalid property \"version\": {s}\n", .{package_json_source.path.text}, ); - Output.flush(); Global.crash(); }; @@ -10254,7 +10250,6 @@ pub const PackageManager = struct { "error: failed to find package in lockfile package index, this is a bug in Bun. Please file a GitHub issue.\n", .{}, ); - Output.flush(); Global.crash(); }) { .PackageID => |id| lockfile.packages.get(id), @@ -10269,7 +10264,6 @@ pub const PackageManager = struct { Output.prettyError("error: could not find package with name: {s}\n", .{ package.name.slice(lockfile.buffers.string_bytes.items), }); - Output.flush(); Global.crash(); }, }; @@ -10316,7 +10310,6 @@ pub const PackageManager = struct { "error: could not find the folder for {s} in node_modules\n", .{pkg_maybe_version_to_patch}, ); - Output.flush(); Global.crash(); }; @@ -10613,7 +10606,6 @@ pub const PackageManager = struct { "error: failed to open root node_modules folder: {}\n", .{e}, ); - Output.flush(); Global.crash(); }, }; @@ -10633,7 +10625,6 @@ pub const PackageManager = struct { "error: failed to read package.json: {}\n", .{e.withPath(package_json_path).toSystemError()}, ); - Output.flush(); Global.crash(); }, } @@ -10659,7 +10650,6 @@ pub const PackageManager = struct { "error: invalid package.json, missing or invalid property \"version\": {s}\n", .{package_json_source.path.text}, ); - Output.flush(); Global.crash(); }; @@ -10672,7 +10662,6 @@ pub const PackageManager = struct { "error: failed to find package in lockfile package index, this is a bug in Bun. Please file a GitHub issue.\n", .{}, ); - Output.flush(); Global.crash(); }) { .PackageID => |id| lockfile.packages.get(id), @@ -10687,7 +10676,6 @@ pub const PackageManager = struct { Output.prettyError("error: could not find package with name: {s}\n", .{ package.name.slice(lockfile.buffers.string_bytes.items), }); - Output.flush(); Global.crash(); }, }; @@ -10718,7 +10706,6 @@ pub const PackageManager = struct { "error: could not find the folder for {s} in node_modules\n", .{argument}, ); - Output.flush(); Global.crash(); }; const changes_dir = bun.path.joinZBuf(pathbuf[0..], &[_][]const u8{ @@ -10760,7 +10747,6 @@ pub const PackageManager = struct { "error: failed to read from cache {}\n", .{e.toSystemError()}, ); - Output.flush(); Global.crash(); }, }; @@ -10775,7 +10761,6 @@ pub const PackageManager = struct { "error: failed to make tempdir {s}\n", .{@errorName(e)}, ); - Output.flush(); Global.crash(); }); @@ -10790,7 +10775,6 @@ pub const PackageManager = struct { "error: failed to open directory {s} {s}\n", .{ new_folder, @errorName(e) }, ); - Output.flush(); Global.crash(); }; defer new_folder_handle.close(); @@ -10810,7 +10794,6 @@ pub const PackageManager = struct { "error: failed to make tempdir {s}\n", .{@errorName(e)}, ); - Output.flush(); Global.crash(); }); @@ -10832,7 +10815,6 @@ pub const PackageManager = struct { "error: failed to open directory {s} {s}\n", .{ new_folder, @errorName(e) }, ); - Output.flush(); Global.crash(); }; defer new_folder_handle.close(); @@ -10855,7 +10837,6 @@ pub const PackageManager = struct { "error: failed to open directory {s} {s}\n", .{ new_folder, @errorName(e) }, ); - Output.flush(); Global.crash(); }; defer new_folder_handle.close(); @@ -10892,7 +10873,6 @@ pub const PackageManager = struct { "error: failed to get cwd path {}\n", .{e}, ); - Output.flush(); Global.crash(); }, }; @@ -10902,7 +10882,6 @@ pub const PackageManager = struct { "error: git must be installed to use `bun patch --commit` \n", .{}, ); - Output.flush(); Global.crash(); }; const paths = bun.patch.gitDiffPreprocessPaths(bun.default_allocator, old_folder, new_folder, false); @@ -10913,7 +10892,6 @@ pub const PackageManager = struct { "error: failed to make diff {s}\n", .{@errorName(e)}, ); - Output.flush(); Global.crash(); }) { .result => |r| r, @@ -10922,7 +10900,6 @@ pub const PackageManager = struct { "error: failed to make diff {}\n", .{e}, ); - Output.flush(); Global.crash(); }, }; @@ -10932,7 +10909,6 @@ pub const PackageManager = struct { "error: failed to make diff {s}\n", .{@errorName(e)}, ); - Output.flush(); Global.crash(); }) { .result => |stdout| stdout, @@ -10959,7 +10935,6 @@ pub const PackageManager = struct { Truncate{ .stderr = stderr }, }, ); - Output.flush(); Global.crash(); }, }; @@ -10991,7 +10966,6 @@ pub const PackageManager = struct { "error: failed to open temp file {}\n", .{e.toSystemError()}, ); - Output.flush(); Global.crash(); }, }; @@ -11002,7 +10976,6 @@ pub const PackageManager = struct { "error: failed to write patch to temp file {}\n", .{e.toSystemError()}, ); - Output.flush(); Global.crash(); } @@ -11032,7 +11005,6 @@ pub const PackageManager = struct { "error: failed to make patches dir {}\n", .{e.toSystemError()}, ); - Output.flush(); Global.crash(); } @@ -11047,7 +11019,6 @@ pub const PackageManager = struct { "error: failed renaming patch file to patches dir {}\n", .{e.toSystemError()}, ); - Output.flush(); Global.crash(); } From 1ac11aab0acefdb81c2ddda79704ed32fee32b43 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:27:42 -0700 Subject: [PATCH 48/51] stuff --- src/install/install.zig | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index 9b52c83a619520..2a751599388c3c 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10172,7 +10172,7 @@ pub const PackageManager = struct { name_and_version, pub fn fromArg(argument: []const u8) PatchArgKind { - if (bun.strings.hasPrefix(argument, "node_modules/")) return .path; + if (bun.strings.hasPrefixComptime(argument, "node_modules/")) return .path; if (bun.path.Platform.auto.isAbsolute(argument) and bun.strings.contains(argument, "node_modules/")) return .path; if (comptime bun.Environment.isWindows) { if (bun.strings.hasPrefix(argument, "node_modules\\")) return .path; @@ -10438,11 +10438,7 @@ pub const PackageManager = struct { var in_file_close = true; defer if (in_file_close) in_file.close(); - const mode = in_file.mode() catch |e| { - Output.prettyError("error: failed to get file mode {s} ", .{@errorName(e)}); - Global.crash(); - }; - var outfile = createFile(destination_dir_, entrypath, .{ .mode = mode }) catch |e| { + var outfile = createFile(destination_dir_, entrypath, .{}) catch |e| { Output.prettyError("error: failed to create file {s} ({s})", .{ entrypath, @errorName(e) }); Global.crash(); }; From 60d173406faa75ae217ee000e7c962050c236461 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:29:55 -0700 Subject: [PATCH 49/51] comment --- src/install/install.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src/install/install.zig b/src/install/install.zig index 2a751599388c3c..2b14e26d25f685 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -4261,6 +4261,7 @@ pub const PackageManager = struct { this.patch_task_fifo.writeItemAssumeCapacity(task); } + /// We need to calculate all the patchfile hashes at the beginning so we don't run into problems with stale hashes pub fn enqueuePatchTaskPre(this: *PackageManager, task: *PatchTask) void { debug("Enqueue patch task pre: 0x{x} {s}", .{ @intFromPtr(task), @tagName(task.callback) }); task.pre = true; From a2c64431eea8cc1663d2150f56d65bfe1d6f2035 Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:42:25 -0700 Subject: [PATCH 50/51] check that it is invalid package id --- src/install/install.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src/install/install.zig b/src/install/install.zig index 2b14e26d25f685..475664850258d2 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10128,6 +10128,7 @@ pub const PackageManager = struct { if (dep.name_hash != name_hash) continue; matches_found += 1; const pkg_id = lockfile.buffers.resolutions.items[dep_id]; + if (pkg_id == invalid_package_id) continue; const pkg = lockfile.packages.get(pkg_id); if (version) |v| { const label = std.fmt.bufPrint(buf[0..], "{}", .{pkg.resolution.fmt(strbuf, .posix)}) catch @panic("Resolution name too long"); From 18e62dd923fbae33ef7d71727e57ee7fe3c2652c Mon Sep 17 00:00:00 2001 From: Zack Radisic <56137411+zackradisic@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:43:44 -0700 Subject: [PATCH 51/51] resolve comments --- src/install/install.zig | 16 +--------------- test/cli/install/bun-install-patch.test.ts | 4 ++++ test/cli/install/bun-patch.test.ts | 4 ++++ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/install/install.zig b/src/install/install.zig index bdaa1c05f7ea79..772e149b75ad81 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -10437,33 +10437,19 @@ pub const PackageManager = struct { Global.crash(); } - var in_file = openFile(entry.dir, basename, .{ .mode = .read_only }) catch |e| { - Output.prettyError("error: opening node_modules file {s} {s} ()", .{ @errorName(e), entrypath }); - Global.crash(); - }; - var in_file_close = true; - defer if (in_file_close) in_file.close(); - var outfile = createFile(destination_dir_, entrypath, .{}) catch |e| { Output.prettyError("error: failed to create file {s} ({s})", .{ entrypath, @errorName(e) }); Global.crash(); }; - var out_file_close = true; - defer if (out_file_close) outfile.close(); + outfile.close(); const infile_path = bun.path.joinStringBufWZ(buf1, &[_][]const u16{ in_dir, entry.path }, .auto); const outfile_path = bun.path.joinStringBufWZ(buf2, &[_][]const u16{ out_dir, entry.path }, .auto); - in_file_close = false; - out_file_close = false; - in_file.close(); - outfile.close(); bun.copyFileWithState(infile_path, outfile_path, ©_file_state) catch |err| { Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); Global.crash(); }; - - continue; } else if (comptime Environment.isPosix) { var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); defer in_file.close(); diff --git a/test/cli/install/bun-install-patch.test.ts b/test/cli/install/bun-install-patch.test.ts index c8375d9647e1e4..332aed1d1c5c88 100644 --- a/test/cli/install/bun-install-patch.test.ts +++ b/test/cli/install/bun-install-patch.test.ts @@ -2,6 +2,10 @@ import { $ } from "bun"; import { bunExe, bunEnv as env, toBeValidBin, toHaveBins, toBeWorkspaceLink, tempDirWithFiles, bunEnv } from "harness"; import { afterAll, afterEach, beforeAll, beforeEach, expect, it, describe, test, setDefaultTimeout } from "bun:test"; +beforeAll(() => { + setDefaultTimeout(1000 * 60 * 5); +}); + describe("patch", async () => { const is_even_patch = /* patch */ `diff --git a/index.js b/index.js index 832d92223a9ec491364ee10dcbe3ad495446ab80..bc652e496c165a7415880ef4520c0ab302bf0765 100644 diff --git a/test/cli/install/bun-patch.test.ts b/test/cli/install/bun-patch.test.ts index a473fbb10b8b6a..e4655b7660c149 100644 --- a/test/cli/install/bun-patch.test.ts +++ b/test/cli/install/bun-patch.test.ts @@ -7,6 +7,10 @@ const expectNoError = (o: ShellOutput) => expect(o.stderr.toString()).not.toCont // const platformPath = (path: string) => (process.platform === "win32" ? path.replaceAll("/", sep) : path); const platformPath = (path: string) => path; +beforeAll(() => { + setDefaultTimeout(1000 * 60 * 5); +}); + describe("bun patch ", async () => { // Tests to make sure that patching describe("popular pkg", async () => {