Feature: breaking: exclude disks option (#1260)

docs/usage/arcade.md

@@ -82,6 +82,12 @@ The ROM merge type can be specified with the `--merge-roms <type>` option:
       --merge-roms split
     ```
 
+## CHD disks
+
+As arcade machines got more complicated, their storage requirements grew beyond what ROM chips can handle cost effectively. Cabinets started embedding hard drives, optical drives, laser disc drives, and more. Because backup images of these media types can get large, the MAME developers created a new compression format called "compressed hunks of data" (CHD).
+
+MAME DATs catalog these "disks" separately from "ROMs", which lets users choose whether to care about them or not. Typically, games that require disks will not run without them, so Igir requires them for a game to be considered present/complete. You can use the `--exclude-disks` option to exclude disks and only process ROMs to save some space.
+
 ## Example: building a new ROM set
 
 Let's say we want to build an arcade ROM set that's compatible with the most recent version of [RetroArch](desktop/retroarch.md). The steps would look like this:

src/igir.ts

@@ -307,17 +307,33 @@ export default class Igir {
     }
 
     dats.forEach((dat) => {
-      const datMinimumBitmask = dat.getRequiredChecksumBitmask();
+      const datMinimumRomBitmask = dat.getRequiredRomChecksumBitmask();
       Object.keys(ChecksumBitmask)
         .filter((bitmask): bitmask is keyof typeof ChecksumBitmask => Number.isNaN(Number(bitmask)))
         // Has not been enabled yet
         .filter((bitmask) => ChecksumBitmask[bitmask] > minimumChecksum)
         .filter((bitmask) => !(matchChecksum & ChecksumBitmask[bitmask]))
         // Should be enabled for this DAT
-        .filter((bitmask) => datMinimumBitmask & ChecksumBitmask[bitmask])
+        .filter((bitmask) => datMinimumRomBitmask & ChecksumBitmask[bitmask])
         .forEach((bitmask) => {
           matchChecksum |= ChecksumBitmask[bitmask];
-          this.logger.trace(`${dat.getNameShort()}: needs ${bitmask} file checksums, enabling`);
+          this.logger.trace(`${dat.getNameShort()}: needs ${bitmask} file checksums for ROMs, enabling`);
+        });
+
+      if (this.options.getExcludeDisks()) {
+        return;
+      }
+      const datMinimumDiskBitmask = dat.getRequiredDiskChecksumBitmask();
+      Object.keys(ChecksumBitmask)
+        .filter((bitmask): bitmask is keyof typeof ChecksumBitmask => Number.isNaN(Number(bitmask)))
+        // Has not been enabled yet
+        .filter((bitmask) => ChecksumBitmask[bitmask] > minimumChecksum)
+        .filter((bitmask) => !(matchChecksum & ChecksumBitmask[bitmask]))
+        // Should be enabled for this DAT
+        .filter((bitmask) => datMinimumDiskBitmask & ChecksumBitmask[bitmask])
+        .forEach((bitmask) => {
+          matchChecksum |= ChecksumBitmask[bitmask];
+          this.logger.trace(`${dat.getNameShort()}: needs ${bitmask} file checksums for disks, enabling`);
         });
     });
 

src/modules/argumentsParser.ts

@@ -79,7 +79,7 @@ export default class ArgumentsParser {
     const groupRomZip = 'zip command options:';
     const groupRomLink = 'link command options:';
     const groupRomHeader = 'ROM header options:';
-    const groupRomSet = 'ROM set options:';
+    const groupRomSet = 'ROM set options (requires DATs):';
     const groupRomFiltering = 'ROM filtering options:';
     const groupRomPriority = 'One game, one ROM (1G1R) options:';
     const groupReport = 'report command options:';
@@ -520,15 +520,30 @@ export default class ArgumentsParser {
         requiresArg: true,
         default: MergeMode[MergeMode.FULLNONMERGED].toLowerCase(),
       })
+      .check((checkArgv) => {
+        // Re-implement `implies: 'dat'`, which isn't possible with a default value
+        if (checkArgv['merge-roms'] !== MergeMode[MergeMode.FULLNONMERGED].toLowerCase() && !checkArgv.dat) {
+          throw new ExpectedError('Missing dependent arguments:\n merge-roms -> dat');
+        }
+        return true;
+      })
+      .option('exclude-disks', {
+        group: groupRomSet,
+        description: 'Exclude CHD disks in DATs from processing & writing',
+        type: 'boolean',
+        implies: 'dat',
+      })
       .option('allow-excess-sets', {
         group: groupRomSet,
         description: 'Allow writing archives that have excess files when not extracting or zipping',
         type: 'boolean',
+        implies: 'dat',
       })
       .option('allow-incomplete-sets', {
         group: groupRomSet,
         description: 'Allow writing games that don\'t have all of their ROMs',
         type: 'boolean',
+        implies: 'dat',
       })
 
       .option('filter-regex', {

src/modules/candidateGenerator.ts

@@ -120,9 +120,14 @@ export default class CandidateGenerator extends Module {
   ): Promise<ReleaseCandidate | undefined> {
     const romsToInputFiles = this.getInputFilesForGame(dat, game, indexedFiles);
 
+    const gameRoms = [
+      ...game.getRoms(),
+      ...(this.options.getExcludeDisks() ? [] : game.getDisks()),
+    ];
+
     // For each Game's ROM, find the matching File
     const romFiles = await Promise.all(
-      game.getRoms().map(async (rom) => {
+      gameRoms.map(async (rom) => {
         if (!romsToInputFiles.has(rom)) {
           return [rom, undefined];
         }
@@ -183,15 +188,18 @@ export default class CandidateGenerator extends Module {
          * Matches {@link ROMHeaderProcessor.getFileWithHeader}
          */
         if (inputFile instanceof ArchiveEntry
-          && !this.options.shouldZipFile(rom.getName())
-          && !this.options.shouldExtract()
+          && !this.options.shouldZipRom(rom)
+          && !this.options.shouldExtractRom(rom)
         ) {
           try {
             // Note: we're delaying checksum calculation for now, {@link CandidateArchiveFileHasher}
             //  will handle it later
             inputFile = new ArchiveFile(
               inputFile.getArchive(),
-              { checksumBitmask: inputFile.getChecksumBitmask() },
+              {
+                size: await fsPoly.size(inputFile.getFilePath()),
+                checksumBitmask: inputFile.getChecksumBitmask(),
+              },
             );
           } catch (error) {
             this.progressBar.logWarn(`${dat.getNameShort()}: ${game.getName()}: ${error}`);
@@ -270,7 +278,12 @@ export default class CandidateGenerator extends Module {
     game: Game,
     indexedFiles: IndexedFiles,
   ): Map<ROM, File> {
-    const romsAndInputFiles = game.getRoms().map((rom) => ([
+    const gameRoms = [
+      ...game.getRoms(),
+      ...(this.options.getExcludeDisks() ? [] : game.getDisks()),
+    ];
+
+    const romsAndInputFiles = gameRoms.map((rom) => ([
       rom,
       indexedFiles.findFiles(rom) ?? [],
     ])) satisfies [ROM, File[]][];
@@ -311,8 +324,8 @@ export default class CandidateGenerator extends Module {
         // If there is a CHD with every .bin file, and we're raw-copying it, then assume its .cue
         // file is accurate
         return archive instanceof Chd
-          && !game.getRoms().some((rom) => this.options.shouldZipFile(rom.getName()))
-          && !this.options.shouldExtract()
+          && !game.getRoms().some((rom) => this.options.shouldZipRom(rom))
+          && !game.getRoms().some((rom) => this.options.shouldExtractRom(rom))
           && CandidateGenerator.onlyCueFilesMissingFromChd(game, roms)
           && this.options.getAllowExcessSets();
       })
@@ -407,7 +420,7 @@ export default class CandidateGenerator extends Module {
     }
 
     // Determine the output file type
-    if (this.options.shouldZipFile(rom.getName())) {
+    if (this.options.shouldZipRom(rom)) {
       // Should zip, return an archive entry within an output zip
       return ArchiveEntry.entryOf({
         archive: new Zip(outputFilePath),

src/modules/candidateWriter.ts

@@ -482,7 +482,7 @@ export default class CandidateWriter extends Module {
     inputRomFile: File,
     outputFilePath: string,
   ): Promise<boolean> {
-    this.progressBar.logInfo(`${dat.getNameShort()}: ${releaseCandidate.getName()}: copying file '${inputRomFile.toString()}' (${fsPoly.sizeReadable(inputRomFile.getSize())}) -> '${outputFilePath}'`);
+    this.progressBar.logInfo(`${dat.getNameShort()}: ${releaseCandidate.getName()}: ${inputRomFile instanceof ArchiveEntry ? 'extracting' : 'copying'} file '${inputRomFile.toString()}' (${fsPoly.sizeReadable(inputRomFile.getSize())}) -> '${outputFilePath}'`);
 
     try {
       await CandidateWriter.ensureOutputDirExists(outputFilePath);

src/modules/datMergerSplitter.ts

@@ -65,6 +65,9 @@ export default class DATMergerSplitter extends Module {
         // Get rid of duplicate ROMs. MAME will sometimes duplicate a file with the exact same
         // name, size, and checksum but with a different "region" (e.g. neogeo).
         .filter(ArrayPoly.filterUniqueMapped((rom) => rom.getName())),
+      disk: game.getDisks()
+        // Get rid of ROMs that haven't been dumped yet
+        .filter((disk) => disk.getStatus() !== 'nodump'),
     }));
 
     // 'full' types expect device ROMs to be included
@@ -115,33 +118,33 @@ export default class DATMergerSplitter extends Module {
           }
 
           return game.withProps({
-            rom: DATMergerSplitter.diffGameRoms(biosGame, game),
+            rom: DATMergerSplitter.diffGameRoms(biosGame.getRoms(), game.getRoms()),
           });
         });
     }
 
-    // 'split' and 'merged' types should exclude ROMs found in their parent
+    // 'split' and 'merged' types should exclude ROMs & disks found in their parent
     if (this.options.getMergeRoms() === MergeMode.SPLIT
       || this.options.getMergeRoms() === MergeMode.MERGED
     ) {
-      games = games
-        .map((game) => {
-          if (!game.getParent()) {
-            // This game doesn't have a parent
-            return game;
-          }
+      games = games.map((game) => {
+        if (!game.getParent()) {
+          // This game doesn't have a parent
+          return game;
+        }
 
-          const parentGame = gameNamesToGames.get(game.getParent());
-          if (!parentGame) {
-            // Invalid cloneOf attribute, parent not found
-            this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()} references an invalid parent: ${game.getParent()}`);
-            return game;
-          }
+        const parentGame = gameNamesToGames.get(game.getParent());
+        if (!parentGame) {
+          // Invalid cloneOf attribute, parent not found
+          this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()} references an invalid parent: ${game.getParent()}`);
+          return game;
+        }
 
-          return game.withProps({
-            rom: DATMergerSplitter.diffGameRoms(parentGame, game),
-          });
+        return game.withProps({
+          rom: DATMergerSplitter.diffGameRoms(parentGame.getRoms(), game.getRoms()),
+          disk: DATMergerSplitter.diffGameRoms(parentGame.getDisks(), game.getDisks()),
         });
+      });
     }
 
     const parentGame = games.find((game) => game.isParent());
@@ -173,13 +176,13 @@ export default class DATMergerSplitter extends Module {
     })];
   }
 
-  private static diffGameRoms(parent: Game, child: Game): ROM[] {
-    const parentRomNamesToHashCodes = parent.getRoms().reduce((map, rom) => {
+  private static diffGameRoms(parentRoms: ROM[], childRoms: ROM[]): ROM[] {
+    const parentRomNamesToHashCodes = parentRoms.reduce((map, rom) => {
       map.set(rom.getName(), rom.hashCode());
       return map;
     }, new Map<string, string>());
 
-    return child.getRoms().filter((rom) => {
+    return childRoms.filter((rom) => {
       const parentName = rom.getMerge() ?? rom.getName();
       const parentHashCode = parentRomNamesToHashCodes.get(parentName);
       if (!parentHashCode) {

src/modules/datScanner.ts

@@ -8,9 +8,15 @@ import DriveSemaphore from '../driveSemaphore.js';
 import ArrayPoly from '../polyfill/arrayPoly.js';
 import bufferPoly from '../polyfill/bufferPoly.js';
 import fsPoly from '../polyfill/fsPoly.js';
-import CMProParser, { DATProps, GameProps, ROMProps } from '../types/dats/cmpro/cmProParser.js';
+import CMProParser, {
+  DATProps,
+  DiskProps,
+  GameProps,
+  ROMProps,
+} from '../types/dats/cmpro/cmProParser.js';
 import DAT from '../types/dats/dat.js';
 import DATObject, { DATObjectProps } from '../types/dats/datObject.js';
+import Disk from '../types/dats/disk.js';
 import Game from '../types/dats/game.js';
 import Header from '../types/dats/logiqx/header.js';
 import LogiqxDAT from '../types/dats/logiqx/logiqxDat.js';
@@ -333,6 +339,8 @@ export default class DATScanner extends Scanner {
     }
 
     const games = cmproDatGames.flatMap((game) => {
+      const gameName = game.name ?? game.comment;
+
       let gameRoms: ROMProps[] = [];
       if (game.rom) {
         if (Array.isArray(game.rom)) {
@@ -341,16 +349,29 @@ export default class DATScanner extends Scanner {
           gameRoms = [game.rom];
         }
       }
-      const gameName = game.name ?? game.comment;
+      const roms = gameRoms.map((entry) => new ROM({
+        name: entry.name ?? '',
+        size: Number.parseInt(entry.size ?? '0', 10),
+        crc32: entry.crc,
+        md5: entry.md5,
+        sha1: entry.sha1,
+      }));
 
-      const roms = gameRoms
-        .map((entry) => new ROM({
-          name: entry.name ?? '',
-          size: Number.parseInt(entry.size ?? '0', 10),
-          crc32: entry.crc,
-          md5: entry.md5,
-          sha1: entry.sha1,
-        }));
+      let gameDisks: DiskProps[] = [];
+      if (game.disk) {
+        if (Array.isArray(game.disk)) {
+          gameDisks = game.disk;
+        } else {
+          gameDisks = [game.disk];
+        }
+      }
+      const disks = gameDisks.map((entry) => new Disk({
+        name: entry.name ?? '',
+        size: Number.parseInt(entry.size ?? '0', 10),
+        crc32: entry.crc,
+        md5: entry.md5,
+        sha1: entry.sha1,
+      }));
 
       return new Game({
         name: gameName,
@@ -365,6 +386,7 @@ export default class DATScanner extends Scanner {
         genre: game.genre?.toString(),
         release: undefined,
         rom: roms,
+        disk: disks,
       });
     });
 

src/types/dats/cmpro/cmProParser.ts

@@ -35,26 +35,26 @@ export interface GameProps extends CMProObject {
   sample?: SampleProps | SampleProps[],
   // NON-STANDARD PROPERTIES
   comment?: string,
-  serial?: string,
-  publisher?: string,
-  releaseyear?: string,
-  releasemonth?: string,
-  developer?: string,
-  users?: string,
-  esrbrating?: string,
+  // serial?: string,
+  // publisher?: string,
+  // releaseyear?: string,
+  // releasemonth?: string,
+  // developer?: string,
+  // users?: string,
+  // esrbrating?: string,
   genre?: string,
 }
 
 export interface ROMProps extends CMProObject {
   name?: string,
-  merge?: string,
+  // merge?: string,
   size?: string,
   crc?: string,
-  flags?: string,
+  // flags?: string,
   md5?: string,
   sha1?: string,
   // NON-STANDARD PROPERTIES
-  serial?: string,
+  // serial?: string,
 }
 
 export interface DiskProps extends ROMProps {}

src/types/dats/dat.ts

@@ -133,7 +133,7 @@ export default abstract class DAT {
     return this.getName().match(/\(headerless\)/i) !== null;
   }
 
-  getRequiredChecksumBitmask(): number {
+  getRequiredRomChecksumBitmask(): number {
     let checksumBitmask = 0;
     this.getGames().forEach((game) => game.getRoms().forEach((rom) => {
       if (rom.getCrc32() && rom.getSize()) {
@@ -149,6 +149,22 @@ export default abstract class DAT {
     return checksumBitmask;
   }
 
+  getRequiredDiskChecksumBitmask(): number {
+    let checksumBitmask = 0;
+    this.getGames().forEach((game) => game.getDisks().forEach((disk) => {
+      if (disk.getCrc32() && disk.getSize()) {
+        checksumBitmask |= ChecksumBitmask.CRC32;
+      } else if (disk.getMd5()) {
+        checksumBitmask |= ChecksumBitmask.MD5;
+      } else if (disk.getSha1()) {
+        checksumBitmask |= ChecksumBitmask.SHA1;
+      } else if (disk.getSha256()) {
+        checksumBitmask |= ChecksumBitmask.SHA256;
+      }
+    }));
+    return checksumBitmask;
+  }
+
   /**
    * Serialize this {@link DAT} to the file contents of an XML file.
    */