Feature: group bin/cue files together when inferring games

src/igir.ts

@@ -114,7 +114,7 @@ export default class Igir {
     // Set up progress bar and input for DAT processing
     const datProcessProgressBar = await this.logger.addProgressBar(chalk.underline('Processing DATs'), ProgressBarSymbol.NONE, dats.length);
     if (dats.length === 0) {
-      dats = new DATGameInferrer(this.options, datProcessProgressBar).infer(roms);
+      dats = await new DATGameInferrer(this.options, datProcessProgressBar).infer(roms);
     }
 
     const datsToWrittenFiles = new Map<DAT, File[]>();

src/modules/candidateGenerator.ts

@@ -233,7 +233,7 @@ export default class CandidateGenerator extends Module {
       const inputChds = foundRomsWithFiles
         .map((romWithFiles) => romWithFiles.getOutputFile())
         .filter((file) => file.getFilePath().toLowerCase().endsWith('.chd'))
-        .filter(ArrayPoly.filterUniqueMapped((file) => file.hashCode()));
+        .filter(ArrayPoly.filterUniqueMapped((file) => file.getFilePath()));
       if (inputChds.length === 1) {
         this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()}: `);
         return new ReleaseCandidate(game, release, foundRomsWithFiles);
@@ -305,15 +305,16 @@ export default class CandidateGenerator extends Module {
     // Filter to the Archives that contain every ROM in this Game
     const archivesWithEveryRom = [...inputArchivesToRoms.entries()]
       .filter(([archive, roms]) => {
-        if (roms.length === game.getRoms().length) {
+        if (roms.map((rom) => rom.hashCode()).join(',') === game.getRoms().map((rom) => rom.hashCode()).join(',')) {
           return true;
         }
         // If there is a CHD with every .bin file, and we're raw-copying it, then assume its .cue
         // file is accurate
         return archive instanceof Chd
           && !game.getRoms().some((rom) => this.options.shouldZipFile(rom.getName()))
           && !this.options.shouldExtract()
-          && CandidateGenerator.onlyCueFilesMissingFromChd(game, roms);
+          && CandidateGenerator.onlyCueFilesMissingFromChd(game, roms)
+          && this.options.getAllowExcessSets();
       })
       .map(([archive]) => archive);
 

src/modules/datGameInferrer.ts

@@ -1,5 +1,8 @@
+import fs from 'node:fs';
 import path from 'node:path';
+import util from 'node:util';
 
+import { parse } from '@gplane/cue';
 import moment from 'moment';
 
 import ProgressBar from '../console/progressBar.js';
@@ -34,7 +37,7 @@ export default class DATGameInferrer extends Module {
   /**
    * Infer {@link Game}s from input files.
    */
-  infer(romFiles: File[]): DAT[] {
+  async infer(romFiles: File[]): Promise<DAT[]> {
     this.progressBar.logTrace(`inferring DATs for ${romFiles.length.toLocaleString()} ROM${romFiles.length !== 1 ? 's' : ''}`);
 
     const normalizedInputPaths = this.options.getInputPaths()
@@ -57,14 +60,14 @@ export default class DATGameInferrer extends Module {
     }, new Map<string, File[]>());
     this.progressBar.logTrace(`inferred ${inputPathsToRomFiles.size.toLocaleString()} DAT${inputPathsToRomFiles.size !== 1 ? 's' : ''}`);
 
-    const dats = [...inputPathsToRomFiles.entries()]
-      .map(([inputPath, datRomFiles]) => DATGameInferrer.createDAT(inputPath, datRomFiles));
+    const dats = await Promise.all([...inputPathsToRomFiles.entries()]
+      .map(async ([inputPath, datRomFiles]) => this.createDAT(inputPath, datRomFiles)));
 
     this.progressBar.logTrace('done inferring DATs');
     return dats;
   }
 
-  private static createDAT(inputPath: string, romFiles: File[]): DAT {
+  private async createDAT(inputPath: string, romFiles: File[]): Promise<DAT> {
     const datName = path.basename(inputPath);
     const date = moment().format('YYYYMMDD-HHmmss');
     const header = new Header({
@@ -80,50 +83,46 @@ export default class DATGameInferrer extends Module {
       ].join('\n'),
     });
 
-    // For all non-archived files, group files of the same filename without extension together
-    const gameNamesToRawFiles = romFiles
-      .filter((file) => !(file instanceof ArchiveEntry))
-      .reduce((map, file) => {
-        const gameName = DATGameInferrer.getGameName(file);
-        map.set(gameName, [...(map.get(gameName) ?? []), file]);
-        return map;
-      }, new Map<string, File[]>());
+    let remainingRomFiles = romFiles;
+    let gameNamesToRomFiles: [string, File[]][] = [];
+
+    // For each inference strategy
+    const inferFunctions = [
+      this.inferArchiveEntries,
+      this.inferBinCueFiles,
+      this.inferRawFiles,
+    ];
+    for (const inferFunction of inferFunctions) {
+      // Infer the games and their files)
+      const result = await inferFunction.bind(this)(remainingRomFiles);
+
+      // Update the list of results
+      gameNamesToRomFiles = [...gameNamesToRomFiles, ...result];
+
+      // Remove the consumed files from further inference
+      const consumedFiles = new Set(result
+        .flatMap(([, resultFiles]) => resultFiles)
+        .map((file) => file.toString()));
+      remainingRomFiles = remainingRomFiles.filter((file) => !consumedFiles.has(file.toString()));
+    }
 
-    // For archives, assume the entire archive is one game
-    const archivePathsToArchiveEntries = romFiles
-      .filter((file) => file instanceof ArchiveEntry)
-      .reduce((map, file) => {
-        const archivePath = file.getFilePath();
-        map.set(archivePath, [...(map.get(archivePath) ?? []), file]);
-        return map;
-      }, new Map<string, ArchiveEntry<Archive>[]>());
-    const gameNamesToArchiveEntries = [...archivePathsToArchiveEntries.values()]
-      .map((archiveEntries) => {
-        const gameName = DATGameInferrer.getGameName(archiveEntries[0]);
-        return [gameName, archiveEntries] satisfies [string, ArchiveEntry<Archive>[]];
+    const games = gameNamesToRomFiles.map(([gameName, gameRomFiles]) => {
+      const roms = gameRomFiles
+        .map((romFile) => new ROM({
+          name: path.basename(romFile.getExtractedFilePath()),
+          size: romFile.getSize(),
+          crc32: romFile.getCrc32(),
+          md5: romFile.getMd5(),
+          sha1: romFile.getSha1(),
+          sha256: romFile.getSha256(),
+        }))
+        .filter(ArrayPoly.filterUniqueMapped((rom) => rom.getName()));
+      return new Game({
+        name: gameName,
+        description: gameName,
+        rom: roms,
       });
-
-    const games = [
-      ...gameNamesToRawFiles.entries(),
-      ...gameNamesToArchiveEntries,
-    ]
-      .map(([gameName, gameRomFiles]) => {
-        const roms = gameRomFiles
-          .map((romFile) => new ROM({
-            name: path.basename(romFile.getExtractedFilePath()),
-            size: romFile.getSize(),
-            crc32: romFile.getCrc32(),
-            md5: romFile.getMd5(),
-            sha1: romFile.getSha1(),
-            sha256: romFile.getSha256(),
-          }))
-          .filter(ArrayPoly.filterUniqueMapped((rom) => rom.getName()));
-        return new Game({
-          name: gameName,
-          description: gameName,
-          rom: roms,
-        });
-      })
+    })
       // Filter out duplicate games
       .filter(ArrayPoly.filterUniqueMapped((game) => game.hashCode()));
 
@@ -143,4 +142,79 @@ export default class DATGameInferrer extends Module {
       .replace(/(\.[a-z0-9]+)+$/, '')
       .trim();
   }
+
+  private inferArchiveEntries(romFiles: File[]): [string, ArchiveEntry<Archive>[]][] {
+    this.progressBar.logTrace(`inferring games from archives from ${romFiles.length.toLocaleString()} file${romFiles.length !== 1 ? 's' : ''}`);
+
+    // For archives, assume the entire archive is one game
+    const archivePathsToArchiveEntries = romFiles
+      .filter((file) => file instanceof ArchiveEntry)
+      .reduce((map, file) => {
+        const archivePath = file.getFilePath();
+        map.set(archivePath, [...(map.get(archivePath) ?? []), file]);
+        return map;
+      }, new Map<string, ArchiveEntry<Archive>[]>());
+
+    const results = [...archivePathsToArchiveEntries.values()]
+      .map((archiveEntries) => {
+        const gameName = DATGameInferrer.getGameName(archiveEntries[0]);
+        return [gameName, archiveEntries] satisfies [string, ArchiveEntry<Archive>[]];
+      });
+
+    this.progressBar.logTrace(`inferred ${results.length.toLocaleString()} games from archives`);
+    return results;
+  }
+
+  private async inferBinCueFiles(romFiles: File[]): Promise<[string, File[]][]> {
+    const rawFiles = romFiles.filter((file) => !(file instanceof ArchiveEntry));
+    this.progressBar.logTrace(`inferring games from cue files from ${rawFiles.length.toLocaleString()} non-archive${rawFiles.length !== 1 ? 's' : ''}`);
+
+    const rawFilePathsToFiles = rawFiles
+      .reduce((map, file) => {
+        map.set(file.getFilePath(), file);
+        return map;
+      }, new Map<string, File>());
+
+    const results = (await Promise.all(rawFiles
+      .filter((file) => file.getExtractedFilePath().toLowerCase().endsWith('.cue'))
+      .map(async (cueFile): Promise<[string, File[]] | undefined> => {
+        try {
+          const cueData = await util.promisify(fs.readFile)(cueFile.getFilePath());
+          const cueSheet = parse(cueData.toString(), {
+            fatal: true,
+          }).sheet;
+
+          const binFiles = cueSheet.files
+            .map((binFile) => path.join(path.dirname(cueFile.getFilePath()), binFile.name))
+            .map((binFilePath) => rawFilePathsToFiles.get(binFilePath))
+            .filter(ArrayPoly.filterNotNullish);
+
+          if (binFiles.length > 0) {
+            const gameName = DATGameInferrer.getGameName(cueFile);
+            return [gameName, [cueFile, ...binFiles]];
+          }
+          return undefined;
+        } catch {
+          return undefined;
+        }
+      }))).filter(ArrayPoly.filterNotNullish);
+
+    this.progressBar.logTrace(`inferred ${results.length.toLocaleString()} games from cue files`);
+    return results;
+  }
+
+  private inferRawFiles(romFiles: File[]): [string, File[]][] {
+    this.progressBar.logTrace(`inferring games from raw files from ${romFiles.length.toLocaleString()} file${romFiles.length !== 1 ? 's' : ''}`);
+
+    const results = romFiles
+      .filter((file) => !(file instanceof ArchiveEntry))
+      .reduce((map, file) => {
+        const gameName = DATGameInferrer.getGameName(file);
+        map.set(gameName, [...(map.get(gameName) ?? []), file]);
+        return map;
+      }, new Map<string, File[]>());
+
+    this.progressBar.logTrace(`inferred ${results.size.toLocaleString()} games from raw files`);
+    return [...results.entries()];
+  }
 }

src/types/files/archives/chd/chdBinCueParser.ts

@@ -73,9 +73,9 @@ export default class ChdBinCueParser {
       // Junk size and checksums because we don't know what it should be
       size: 0,
       crc32: checksumBitmask & ChecksumBitmask.CRC32 ? 'x'.repeat(8) : undefined,
-      md5: checksumBitmask & ChecksumBitmask.CRC32 ? 'x'.repeat(32) : undefined,
-      sha1: checksumBitmask & ChecksumBitmask.CRC32 ? 'x'.repeat(40) : undefined,
-      sha256: checksumBitmask & ChecksumBitmask.CRC32 ? 'x'.repeat(64) : undefined,
+      md5: checksumBitmask & ChecksumBitmask.MD5 ? 'x'.repeat(32) : undefined,
+      sha1: checksumBitmask & ChecksumBitmask.SHA1 ? 'x'.repeat(40) : undefined,
+      sha256: checksumBitmask & ChecksumBitmask.SHA256 ? 'x'.repeat(64) : undefined,
     });
 
     return [cueFile, ...binFiles];

test/igir.test.ts

@@ -165,10 +165,10 @@ describe('with explicit DATs', () => {
         [path.join('Headerless', 'allpads.nes'), '6339abe6'],
         [path.join('Headerless', 'color_test.nes'), 'c9c1b7aa'],
         [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM (Track 1).bin`, '49ca35fb'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM (Track 2).bin`, '0316f720'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM (Track 3).bin`, 'a320af40'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM.cue`, 'xxxxxxxx'],
+        [path.join('One', 'CD-ROM', 'CD-ROM (Track 1).bin'), '49ca35fb'],
+        [path.join('One', 'CD-ROM', 'CD-ROM (Track 2).bin'), '0316f720'],
+        [path.join('One', 'CD-ROM', 'CD-ROM (Track 3).bin'), 'a320af40'],
+        [path.join('One', 'CD-ROM', 'CD-ROM.cue'), '4ce39e73'],
         [path.join('One', 'Fizzbuzz.nes'), '370517b5'],
         [path.join('One', 'Foobar.lnx'), 'b22c9747'],
         [`${path.join('One', 'GameCube NKit ISO.nkit.iso')}|GameCube NKit ISO.iso`, '5bc2ce5b'],
@@ -218,10 +218,10 @@ describe('with explicit DATs', () => {
       });
 
       expect(result.outputFilesAndCrcs).toEqual([
-        ['CD-ROM.chd|CD-ROM (Track 1).bin', '49ca35fb'],
-        ['CD-ROM.chd|CD-ROM (Track 2).bin', '0316f720'],
-        ['CD-ROM.chd|CD-ROM (Track 3).bin', 'a320af40'],
-        ['CD-ROM.chd|CD-ROM.cue', 'xxxxxxxx'],
+        [path.join('CD-ROM', 'CD-ROM (Track 1).bin'), '49ca35fb'],
+        [path.join('CD-ROM', 'CD-ROM (Track 2).bin'), '0316f720'],
+        [path.join('CD-ROM', 'CD-ROM (Track 3).bin'), 'a320af40'],
+        [path.join('CD-ROM', 'CD-ROM.cue'), '4ce39e73'],
         // Fizzbuzz.nes is explicitly missing!
         ['Foobar.lnx', 'b22c9747'],
         ['GameCube NKit ISO.nkit.iso|GameCube NKit ISO.iso', '5bc2ce5b'],
@@ -273,16 +273,16 @@ describe('with explicit DATs', () => {
 
       expect(result.outputFilesAndCrcs).toEqual([
         [`${path.join('7z', 'Headered', 'diagnostic_test_cartridge.a78.7z')}|diagnostic_test_cartridge.a78`, 'f6cc9b1c'],
-        [`${path.join('chd', 'One', 'CD-ROM.chd')}|CD-ROM (Track 1).bin`, '49ca35fb'],
-        [`${path.join('chd', 'One', 'CD-ROM.chd')}|CD-ROM (Track 2).bin`, '0316f720'],
-        [`${path.join('chd', 'One', 'CD-ROM.chd')}|CD-ROM (Track 3).bin`, 'a320af40'],
-        [`${path.join('chd', 'One', 'CD-ROM.chd')}|CD-ROM.cue`, 'xxxxxxxx'],
+        [path.join('bin', 'One', 'CD-ROM', 'CD-ROM (Track 1).bin'), '49ca35fb'],
+        [path.join('bin', 'One', 'CD-ROM', 'CD-ROM (Track 2).bin'), '0316f720'],
+        [path.join('bin', 'One', 'CD-ROM', 'CD-ROM (Track 3).bin'), 'a320af40'],
         [`${path.join('chd', 'One', 'GD-ROM.chd')}|track.gdi`, 'f16f621c'],
         [`${path.join('chd', 'One', 'GD-ROM.chd')}|track01.bin`, '9796ed9a'],
         [`${path.join('chd', 'One', 'GD-ROM.chd')}|track02.raw`, 'abc178d5'],
         [`${path.join('chd', 'One', 'GD-ROM.chd')}|track03.bin`, '61a363f1'],
         [`${path.join('chd', 'One', 'GD-ROM.chd')}|track04.bin`, 'fc5ff5a0'],
         [path.join('cso', 'four.cso'), '00000000'], // explicitly not deleted, there were no input files with the extension "cso"
+        [path.join('cue', 'One', 'CD-ROM', 'CD-ROM.cue'), '4ce39e73'],
         [`${path.join('gz', 'Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'],
         [`${path.join('gz', 'Patchable', 'Best.gz')}|best.rom`, '1e3d78cf'],
         [`${path.join('iso', 'One', 'GameCube NKit ISO.nkit.iso')}|GameCube NKit ISO.iso`, '5bc2ce5b'],
@@ -716,10 +716,10 @@ describe('with explicit DATs', () => {
         [`${path.join('Headered', 'speed_test_v51.smc')} -> ${path.join('<input>', 'headered', 'speed_test_v51.smc')}`, '9adca6cc'],
         [`${path.join('Headerless', 'color_test.nes')} -> ${path.join('<input>', 'headered', 'color_test.nintendoentertainmentsystem')}`, 'c9c1b7aa'],
         [`${path.join('Headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc -> ${path.join('<input>', 'headerless', 'speed_test_v51.sfc.gz')}|speed_test_v51.sfc`, '8beffd94'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM (Track 1).bin -> ${path.join('<input>', 'chd', 'CD-ROM.chd')}|CD-ROM (Track 1).bin`, '49ca35fb'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM (Track 2).bin -> ${path.join('<input>', 'chd', 'CD-ROM.chd')}|CD-ROM (Track 2).bin`, '0316f720'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM (Track 3).bin -> ${path.join('<input>', 'chd', 'CD-ROM.chd')}|CD-ROM (Track 3).bin`, 'a320af40'],
-        [`${path.join('One', 'CD-ROM.chd')}|CD-ROM.cue -> ${path.join('<input>', 'chd', 'CD-ROM.chd')}|CD-ROM.cue`, 'xxxxxxxx'],
+        [`${path.join('One', 'CD-ROM', 'CD-ROM (Track 1).bin')} -> ${path.join('<input>', 'discs', 'CD-ROM (Track 1).bin')}`, '49ca35fb'],
+        [`${path.join('One', 'CD-ROM', 'CD-ROM (Track 2).bin')} -> ${path.join('<input>', 'discs', 'CD-ROM (Track 2).bin')}`, '0316f720'],
+        [`${path.join('One', 'CD-ROM', 'CD-ROM (Track 3).bin')} -> ${path.join('<input>', 'discs', 'CD-ROM (Track 3).bin')}`, 'a320af40'],
+        [`${path.join('One', 'CD-ROM', 'CD-ROM.cue')} -> ${path.join('<input>', 'discs', 'CD-ROM.cue')}`, '4ce39e73'],
         [`${path.join('One', 'Fizzbuzz.nes')} -> ${path.join('<input>', 'raw', 'fizzbuzz.nes')}`, '370517b5'],
         [`${path.join('One', 'Foobar.lnx')} -> ${path.join('<input>', 'foobar.lnx')}`, 'b22c9747'],
         [`${path.join('One', 'GameCube NKit ISO.nkit.iso')}|GameCube NKit ISO.iso -> ${path.join('<input>', 'nkit', '5bc2ce5b.nkit.iso')}|GameCube NKit ISO.iso`, '5bc2ce5b'],
@@ -1009,7 +1009,10 @@ describe('with inferred DATs', () => {
       const result = await runIgir({
         commands: ['move', 'extract', 'test'],
         input: [path.join(inputTemp, 'roms')],
-        inputExclude: [path.join(inputTemp, 'roms', 'nkit')],
+        inputExclude: [
+          path.join(inputTemp, 'roms', 'chd'),
+          path.join(inputTemp, 'roms', 'nkit'),
+        ],
         output: outputTemp,
         dirGameSubdir: GameSubdirMode[GameSubdirMode.MULTIPLE].toLowerCase(),
         fixExtension: FixExtension[FixExtension.AUTO].toLowerCase(),
@@ -1025,14 +1028,10 @@ describe('with inferred DATs', () => {
         ['before.rom', '0361b321'],
         ['best.rom', '1e3d78cf'],
         ['C01173E.rom', 'dfaebe28'],
-        ['CD-ROM (Track 1).bin', '49ca35fb'],
-        ['CD-ROM (Track 2).bin', '0316f720'],
-        ['CD-ROM (Track 3).bin', 'a320af40'],
-        ['CD-ROM.cue', '4ce39e73'],
         [path.join('CD-ROM', 'CD-ROM (Track 1).bin'), '49ca35fb'],
         [path.join('CD-ROM', 'CD-ROM (Track 2).bin'), '0316f720'],
         [path.join('CD-ROM', 'CD-ROM (Track 3).bin'), 'a320af40'],
-        [path.join('CD-ROM', 'CD-ROM.cue'), '03924922'], // doesn't match the DAT CRC
+        [path.join('CD-ROM', 'CD-ROM.cue'), '4ce39e73'],
         ['color_test.nes', 'c9c1b7aa'],
         ['diagnostic_test_cartridge.a78', 'f6cc9b1c'],
         ['empty.rom', '00000000'],
@@ -1044,11 +1043,6 @@ describe('with inferred DATs', () => {
         [path.join('fourfive', 'five.rom'), '3e5daf67'],
         [path.join('fourfive', 'four.rom'), '1cf3ca74'],
         ['GD-ROM.gdi', 'f16f621c'],
-        [path.join('GD-ROM', 'track.gdi'), 'f16f621c'],
-        [path.join('GD-ROM', 'track01.bin'), '9796ed9a'],
-        [path.join('GD-ROM', 'track02.raw'), 'abc178d5'],
-        [path.join('GD-ROM', 'track03.bin'), '61a363f1'],
-        [path.join('GD-ROM', 'track04.bin'), 'fc5ff5a0'],
         ['KDULVQN.rom', 'b1c303e4'],
         ['LCDTestROM.lnx', '2d251538'],
         ['loremipsum.rom', '70856527'],

test/modules/candidateCombiner.test.ts

@@ -17,7 +17,7 @@ async function runCombinedCandidateGenerator(
   romFiles: File[],
 ): Promise<Map<Parent, ReleaseCandidate[]>> {
   // Run DATGameInferrer, but condense all DATs down to one
-  const dats = new DATGameInferrer(options, new ProgressBarFake()).infer(romFiles);
+  const dats = await new DATGameInferrer(options, new ProgressBarFake()).infer(romFiles);
   const dat = new DATCombiner(new ProgressBarFake()).combine(dats);
 
   const indexedRomFiles = await new ROMIndexer(options, new ProgressBarFake()).index(romFiles);