From 411f33c66f60f72c8f58dbe9c2fa4d9a5b601e6d Mon Sep 17 00:00:00 2001 From: Christian Emmer <10749361+emmercm@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:58:06 -0700 Subject: [PATCH 1/2] Prefer input files on the output disk --- src/modules/fileIndexer.ts | 25 +++++++++++++++++++++---- src/polyfill/fsPoly.ts | 11 ++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/modules/fileIndexer.ts b/src/modules/fileIndexer.ts index c10d1c17f..a2d7930d0 100644 --- a/src/modules/fileIndexer.ts +++ b/src/modules/fileIndexer.ts @@ -1,6 +1,7 @@ import path from 'node:path'; import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js'; +import FsPoly from '../polyfill/fsPoly.js'; import ArchiveEntry from '../types/files/archives/archiveEntry.js'; import Rar from '../types/files/archives/rar.js'; import SevenZip from '../types/files/archives/sevenZip.js'; @@ -48,6 +49,9 @@ export default class FileIndexer extends Module { } }); + const outputDir = path.resolve(this.options.getOutputDirRoot()); + const outputDirDisk = FsPoly.disksSync().find((mount) => outputDir.startsWith(mount)); + // Sort the file arrays [...results.entries()] .forEach(([hashCode, filesForHash]) => filesForHash.sort((fileOne, fileTwo) => { @@ -67,14 +71,27 @@ export default class FileIndexer extends Module { return fileOneArchived - fileTwoArchived; } - // Then, prefer files that are already in the output directory - const outputDir = path.resolve(this.options.getOutputDirRoot()); - const fileOneInOutput = path.resolve(fileOne.getFilePath()).startsWith(outputDir) ? 0 : 1; - const fileTwoInOutput = path.resolve(fileTwo.getFilePath()).startsWith(outputDir) ? 0 : 1; + // Then, prefer files that are NOT already in the output directory + // This is in case the output file is invalid and we're trying to overwrite it with + // something else. Otherwise, we'll just attempt to overwrite the invalid output file with + // itself, still resulting in an invalid output file. + const fileOneInOutput = path.resolve(fileOne.getFilePath()).startsWith(outputDir) ? 1 : 0; + const fileTwoInOutput = path.resolve(fileTwo.getFilePath()).startsWith(outputDir) ? 1 : 0; if (fileOneInOutput !== fileTwoInOutput) { return fileOneInOutput - fileTwoInOutput; } + // Then, prefer files that are on the same disk for fs efficiency see {@link FsPoly#mv} + if (outputDirDisk) { + const fileOneInOutputDisk = path.resolve(fileOne.getFilePath()) + .startsWith(outputDirDisk) ? 0 : 1; + const fileTwoInOutputDisk = path.resolve(fileTwo.getFilePath()) + .startsWith(outputDirDisk) ? 0 : 1; + if (fileOneInOutputDisk !== fileTwoInOutputDisk) { + return fileOneInOutputDisk - fileTwoInOutputDisk; + } + } + // Otherwise, be deterministic return fileOne.getFilePath().localeCompare(fileTwo.getFilePath()); })); diff --git a/src/polyfill/fsPoly.ts b/src/polyfill/fsPoly.ts index 761245395..81046ce77 100644 --- a/src/polyfill/fsPoly.ts +++ b/src/polyfill/fsPoly.ts @@ -13,9 +13,7 @@ export default class FsPoly { static readonly FILE_READING_CHUNK_SIZE = 1024 * 1024; // 1MiB // Assume that all drives we're reading from or writing to were already mounted at startup - private static readonly DRIVE_MOUNTS = nodeDiskInfo.getDiskInfoSync() - .map((info) => info.mounted) - .sort((a, b) => b.split(/[\\/]/).length - a.split(/[\\/]/).length); + public static readonly DRIVE_MOUNTS = FsPoly.disksSync(); static async canSymlink(tempDir: string): Promise { const source = await this.mktemp(path.join(tempDir, 'source')); @@ -57,6 +55,13 @@ export default class FsPoly { } } + static disksSync(): string[] { + return nodeDiskInfo.getDiskInfoSync() + .filter((info) => info.available > 0) + .map((info) => info.mounted) + .sort((a, b) => b.split(/[\\/]/).length - a.split(/[\\/]/).length); + } + /** * There is no promise version of existsSync() */ From c17988d452fe3862671ea776707681988675d263 Mon Sep 17 00:00:00 2001 From: Christian Emmer <10749361+emmercm@users.noreply.github.com> Date: Sat, 28 Oct 2023 17:03:58 -0700 Subject: [PATCH 2/2] Merge fix --- src/modules/fileIndexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/fileIndexer.ts b/src/modules/fileIndexer.ts index a2d7930d0..03bcbbe27 100644 --- a/src/modules/fileIndexer.ts +++ b/src/modules/fileIndexer.ts @@ -50,7 +50,7 @@ export default class FileIndexer extends Module { }); const outputDir = path.resolve(this.options.getOutputDirRoot()); - const outputDirDisk = FsPoly.disksSync().find((mount) => outputDir.startsWith(mount)); + const outputDirDisk = (await FsPoly.disks()).find((mount) => outputDir.startsWith(mount)); // Sort the file arrays [...results.entries()]