Skip to content

Commit

Permalink
Feature: CSO/ZSO/DAX support (#1253)
Browse files Browse the repository at this point in the history
  • Loading branch information
emmercm authored Jul 30, 2024
1 parent e16494d commit f86f5d4
Show file tree
Hide file tree
Showing 30 changed files with 294 additions and 131 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,9 @@ site/
*.cd2
*.chd
*.col
*.cso
*.cue
*.dax
*.dvd
*.gb
*.gba
Expand Down Expand Up @@ -159,6 +161,7 @@ site/
*.zim
*.z64
*.zip
*.zso

# ROM pack excess
*.bmp
Expand Down
3 changes: 2 additions & 1 deletion docs/alternatives.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ There are a few different popular ROM managers that have similar features:
| DATs: fixdat creation |[Fixdat docs](dats/fixdats.md) ||||
| DATs: combine multiple |||||
| Archives: extraction formats | ✅ many formats ([reading archives docs](input/reading-archives.md)) |`.zip`, `.7z`, `.rar` | ⚠️ `.zip`, `.7z` | ⚠️ `.zip`, `.7z` |
| Archives: `.chd` support | ⚠️ via chdman (bundled) | ⚠️ via chdman | ✅ v1-5 natively | ⚠️ v1-4 natively |
| Archives: `.chd` support | ⚠️ via `chdman` (bundled) | ⚠️ via `chdman` | ✅ v1-5 natively | ⚠️ v1-4 natively |
| Archives: `.cso` & `.zso` support | ⚠️ via `maxcso` (bundled) ||||
| Archives: `.nkit.iso` support | ⚠️ matching but no extraction [GameCube docs](usage/console/gamecube.md#nkit) ||||
| Archives: creation formats |`.zip` only by design ([writing archives docs](output/writing-archives.md)) |`.zip`, `.7z`, `.rar` | ⚠️ `.zip` (TorrentZip), `.7z` | ⚠️ `.zip`, `.7z` |
| Archives: automatic extension correction |||||
Expand Down
25 changes: 13 additions & 12 deletions docs/input/reading-archives.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,19 @@ Igir supports scanning the contents of archives for ROMs, DATs, and ROM patches.
Igir supports most common archive formats:

| Extension | Contains file CRC32s | Igir can extract without a third-party binary | Igir can checksum without temporary files |
|------------------------------------------------------------------|----------------------|-------------------------------------------------|---------------------------------------------|
| `.7z` ||`7za` ||
| `.chd` | ❌ SHA1 |`chdman` ||
| `.gz`, `.gzip` | ❌ CRC16 |`7za` ||
| `.nkit.iso` ([GameCube docs](../usage/console/gamecube.md#nkit)) || ❌ no extraction support ||
| `.rar` ||||
| `.tar` ||| ✅ ≤64MiB |
| `.tar.gz`, `.tgz` ||| ✅ ≤64MiB |
| `.z01` ||`7za` ||
| `.zip` (including zip64) ||| ✅ ≤64MiB |
| `.zip.001` ||`7za` ||
| `.zipx` ||`7za` ||
|------------------------------------------------------------------|----------------------|-----------------------------------------------|-------------------------------------------|
| `.7z` ||`7za` ||
| `.chd` | ❌ SHA1 |`chdman` ||
| `.cso`, `.zso`, `.dax` ||`maxcso` | ⚠️ CRC32 only |
| `.gz`, `.gzip` | ❌ CRC16 |`7za` ||
| `.nkit.iso` ([GameCube docs](../usage/console/gamecube.md#nkit)) || ❌ no extraction support ||
| `.rar` ||||
| `.tar` ||| ✅ ≤64MiB |
| `.tar.gz`, `.tgz` ||| ✅ ≤64MiB |
| `.z01` ||`7za` ||
| `.zip` (including zip64) ||| ✅ ≤64MiB |
| `.zip.001` ||`7za` ||
| `.zipx` ||`7za` ||

**You should prefer archive formats that have CRC32 checksum information for each file.**

Expand Down
18 changes: 18 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
"graceful-fs": "4.2.11",
"is-admin": "4.0.0",
"junk": "4.0.1",
"maxcso": "0.1130.5",
"micromatch": "4.0.7",
"moment": "2.30.1",
"node-disk-info": "1.3.0",
Expand Down
3 changes: 3 additions & 0 deletions package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ const fileFilter = (filters: FileFilter[]): string[] => {
// Only include the exact chdman bin we need
{ exclude: 'node_modules/{**/,}chdman/bin/*/*/chdman*' },
{ include: `node_modules/{**/,}chdman/bin/${process.platform}/${process.arch}/chdman*` },
// Only include the exact maxcso bin we need
{ exclude: 'node_modules/{**/,}maxcso/bin/*/*/maxcso*' },
{ include: `node_modules/{**/,}maxcso/bin/${process.platform}/${process.arch}/maxcso*` },
]));
const includeSize = (await Promise.all([...include].map(async (file) => {
if (await FsPoly.isDirectory(file)) {
Expand Down
28 changes: 18 additions & 10 deletions src/modules/candidateGenerator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -318,16 +318,15 @@ export default class CandidateGenerator extends Module {
// Filter to the Archives that contain every ROM in this Game
const archivesWithEveryRom = [...inputArchivesToRoms.entries()]
.filter(([archive, roms]) => {
if (roms.map((rom) => rom.hashCode()).join(',') === game.getRoms().map((rom) => rom.hashCode()).join(',')) {
if (roms.map((rom) => rom.hashCode()).join(',') === gameRoms.map((rom) => rom.hashCode()).join(',')) {
return true;
}
// If there is a CHD with every .bin file, and we're raw-copying it, then assume its .cue
// file is accurate
return archive instanceof Chd
&& !game.getRoms().some((rom) => this.options.shouldZipRom(rom))
&& !game.getRoms().some((rom) => this.options.shouldExtractRom(rom))
&& CandidateGenerator.onlyCueFilesMissingFromChd(game, roms)
&& this.options.getAllowExcessSets();
&& !gameRoms.some((rom) => this.options.shouldZipRom(rom))
&& !gameRoms.some((rom) => this.options.shouldExtractRom(rom))
&& CandidateGenerator.onlyCueFilesMissingFromChd(game, roms);
})
.map(([archive]) => archive);

Expand Down Expand Up @@ -358,7 +357,7 @@ export default class CandidateGenerator extends Module {
if (
// The Game has zero or one ROM, therefore, we don't really care where the file comes from,
// and we should respect any previous sorting of the input files
game.getRoms().length <= 1
gameRoms.length <= 1
// No input archive contains every ROM from this Game
|| archiveWithEveryRom === undefined
// We're extracting files, therefore, we don't really care where the file comes from, and we
Expand All @@ -374,10 +373,19 @@ export default class CandidateGenerator extends Module {
// For each of this Game's ROMs, find the matching ArchiveEntry from this Archive
return new Map(romsAndInputFiles.map(([rom, inputFiles]) => {
this.progressBar.logTrace(`${dat.getNameShort()}: ${game.getName()}: preferring input archive that contains every ROM: ${archiveWithEveryRom.getFilePath()}`);
const archiveEntry = inputFiles.find((
inputFile,
) => inputFile.getFilePath() === archiveWithEveryRom.getFilePath()) as File;
return [rom, archiveEntry];
let archiveEntry = inputFiles
.find((inputFile) => inputFile.getFilePath() === archiveWithEveryRom.getFilePath());

if (!archiveEntry
&& rom.getName().toLowerCase().endsWith('.cue')
&& archiveWithEveryRom instanceof Chd
) {
// We assumed this CHD was fine above, find its .cue file
archiveEntry = (filesByPath.get(archiveWithEveryRom.getFilePath()) ?? [])
.find((file) => file.getExtractedFilePath().toLowerCase().endsWith('.cue'));
}

return [rom, archiveEntry as File];
}));
}

Expand Down
6 changes: 6 additions & 0 deletions src/modules/romIndexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import path from 'node:path';
import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js';
import FsPoly from '../polyfill/fsPoly.js';
import ArchiveEntry from '../types/files/archives/archiveEntry.js';
import Chd from '../types/files/archives/chd/chd.js';
import Maxcso from '../types/files/archives/maxcso/maxcso.js';
import Rar from '../types/files/archives/rar.js';
import SevenZip from '../types/files/archives/sevenZip/sevenZip.js';
import Tar from '../types/files/archives/tar.js';
Expand Down Expand Up @@ -97,6 +99,10 @@ export default class ROMIndexer extends Module {
return 3;
} if (file.getArchive() instanceof SevenZip) {
return 4;
} if (file.getArchive() instanceof Maxcso) {
return 5;
} if (file.getArchive() instanceof Chd) {
return 6;
}
return 99;
}
Expand Down
4 changes: 2 additions & 2 deletions src/polyfill/fsPoly.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,11 +253,11 @@ export default class FsPoly {
}

// Backoff with jitter
if (attempt >= 3) {
if (attempt >= 5) {
throw error;
}
await new Promise((resolve) => {
setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 100));
setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 10));
});

// Attempt to resolve Windows' "EBUSY: resource busy or locked"
Expand Down
2 changes: 1 addition & 1 deletion src/types/files/archives/chd/chdGdiParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export default class ChdGdiParser {
checksumBitmask: number,
): Promise<ArchiveEntry<T>[]> {
const tempDir = await FsPoly.mkdtemp(path.join(Temp.getTempDir(), 'chd-gdi'));
const gdiFilePath = path.join(tempDir, 'track.gdi');
const gdiFilePath = path.join(tempDir, `${path.parse(archive.getFilePath()).name}.gdi`);
let binRawFilePaths: string[] = [];

try {
Expand Down
18 changes: 18 additions & 0 deletions src/types/files/archives/maxcso/cso.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import Archive from '../archive.js';
import Maxcso from './maxcso.js';

export default class Cso extends Maxcso {
// eslint-disable-next-line class-methods-use-this
protected new(filePath: string): Archive {
return new Cso(filePath);
}

static getExtensions(): string[] {
return ['.cso'];
}

// eslint-disable-next-line class-methods-use-this
getExtension(): string {
return Cso.getExtensions()[0];
}
}
18 changes: 18 additions & 0 deletions src/types/files/archives/maxcso/dax.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import Archive from '../archive.js';
import Maxcso from './maxcso.js';

export default class Dax extends Maxcso {
// eslint-disable-next-line class-methods-use-this
protected new(filePath: string): Archive {
return new Dax(filePath);
}

static getExtensions(): string[] {
return ['.dax'];
}

// eslint-disable-next-line class-methods-use-this
getExtension(): string {
return Dax.getExtensions()[0];
}
}
28 changes: 28 additions & 0 deletions src/types/files/archives/maxcso/maxcso.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import path from 'node:path';

import maxcso from 'maxcso';

import Archive from '../archive.js';
import ArchiveEntry from '../archiveEntry.js';

export default abstract class Maxcso extends Archive {
async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]> {
const entryPath = `${path.parse(this.getFilePath()).name}.iso`;
const size = (await maxcso.header(this.getFilePath())).uncompressedSize;
const crc32 = await maxcso.uncompressedCrc32(this.getFilePath());

return [await ArchiveEntry.entryOf({
archive: this,
entryPath,
size: Number(size),
crc32,
}, checksumBitmask)];
}

async extractEntryToFile(entryPath: string, extractedFilePath: string): Promise<void> {
return maxcso.decompress({
inputFilename: this.getFilePath(),
outputFilename: extractedFilePath,
});
}
}
18 changes: 18 additions & 0 deletions src/types/files/archives/maxcso/zso.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import Archive from '../archive.js';
import Maxcso from './maxcso.js';

export default class Zso extends Maxcso {
// eslint-disable-next-line class-methods-use-this
protected new(filePath: string): Archive {
return new Zso(filePath);
}

static getExtensions(): string[] {
return ['.zso'];
}

// eslint-disable-next-line class-methods-use-this
getExtension(): string {
return Zso.getExtensions()[0];
}
}
14 changes: 10 additions & 4 deletions src/types/files/archives/sevenZip/sevenZip.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,23 @@ export default class SevenZip extends Archive {
async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]> {
/**
* WARN(cemmer): even with the above mutex, {@link _7z.list} will still sometimes return no
* entries. Most archives contain at least one file, so assume this is wrong and attempt
* again up to 3 times total.
* entries. This seems to happen more on older Node.js versions (v16, v18) and specific OSes
* (Linux). Most archives contain at least one file, so assume this is wrong and attempt again
* up to 5 times total.
*/
for (let attempt = 1; attempt <= 3; attempt += 1) {
const maxAttempts = 5;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const archiveEntries = await this.getArchiveEntriesNotCached(checksumBitmask);
if (archiveEntries.length > 0) {
return archiveEntries;
}

// Backoff with jitter
if (attempt >= maxAttempts) {
break;
}
await new Promise((resolve) => {
setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 100));
setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 10));
});
}

Expand Down
12 changes: 12 additions & 0 deletions src/types/files/fileCache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,12 @@ export default class FileCache {
}

static async getOrComputeFileHeader(file: File): Promise<ROMHeader | undefined> {
if (!this.enabled) {
return file.createReadStream(
async (stream) => ROMHeader.headerFromFileStream(stream),
);
}

// NOTE(cemmer): we're explicitly not catching ENOENT errors here, we want it to bubble up
const stats = await FsPoly.stat(file.getFilePath());
const cacheKey = this.getCacheKey(file.toString(), ValueType.ROM_HEADER);
Expand Down Expand Up @@ -236,6 +242,12 @@ export default class FileCache {
}

static async getOrComputeFileSignature(file: File): Promise<FileSignature | undefined> {
if (!this.enabled) {
return file.createReadStream(
async (stream) => FileSignature.signatureFromFileStream(stream),
);
}

// NOTE(cemmer): we're explicitly not catching ENOENT errors here, we want it to bubble up
const stats = await FsPoly.stat(file.getFilePath());
const cacheKey = this.getCacheKey(file.toString(), ValueType.FILE_SIGNATURE);
Expand Down
Loading

0 comments on commit f86f5d4

Please sign in to comment.