Skip to content

Commit

Permalink
Draft
Browse files Browse the repository at this point in the history
  • Loading branch information
@emmercm
emmercm committed Mar 8, 2024
1 parent 38fcbe8 commit 2f92d86
Show file tree
Hide file tree
Showing 8 changed files with 214 additions and 99 deletions.
9 changes: 4 additions & 5 deletions src/igir.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import DAT from './types/dats/dat.js';
import Parent from './types/dats/parent.js';
import DATStatus from './types/datStatus.js';
import File from './types/files/file.js';
import IndexedFiles from './types/indexedFiles.js';
import Options from './types/options.js';
import OutputFactory from './types/outputFactory.js';
import Patch from './types/patches/patch.js';
Expand Down Expand Up @@ -73,9 +74,7 @@ export default class Igir {
// Scan and process input files
let dats = await this.processDATScanner();
const indexedRoms = await this.processROMScanner();
const roms = [...indexedRoms.values()]
.flat()
.reduce(ArrayPoly.reduceUnique(), []);
const roms = indexedRoms.getFiles();
const patches = await this.processPatchScanner();

// Set up progress bar and input for DAT processing
Expand Down Expand Up @@ -218,7 +217,7 @@ export default class Igir {
return dats;
}

private async processROMScanner(): Promise<Map<string, File[]>> {
private async processROMScanner(): Promise<IndexedFiles> {
const romScannerProgressBarName = 'Scanning for ROMs';
const romProgressBar = await this.logger.addProgressBar(romScannerProgressBarName);

Expand Down Expand Up @@ -254,7 +253,7 @@ export default class Igir {
private async generateCandidates(
progressBar: ProgressBar,
dat: DAT,
indexedRoms: Map<string, File[]>,
indexedRoms: IndexedFiles,
patches: Patch[],
): Promise<Map<Parent, ReleaseCandidate[]>> {
const candidates = await new CandidateGenerator(this.options, progressBar)
Expand Down
24 changes: 14 additions & 10 deletions src/modules/candidateGenerator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import Archive from '../types/files/archives/archive.js';
import ArchiveEntry from '../types/files/archives/archiveEntry.js';
import Zip from '../types/files/archives/zip.js';
import File from '../types/files/file.js';
import IndexedFiles from '../types/indexedFiles.js';
import Options from '../types/options.js';
import OutputFactory from '../types/outputFactory.js';
import ReleaseCandidate from '../types/releaseCandidate.js';
Expand All @@ -37,9 +38,9 @@ export default class CandidateGenerator extends Module {
*/
async generate(
dat: DAT,
hashCodeToInputFiles: Map<string, File[]>,
indexedFiles: IndexedFiles,
): Promise<Map<Parent, ReleaseCandidate[]>> {
if (hashCodeToInputFiles.size === 0) {
if (indexedFiles.getFiles().length === 0) {
this.progressBar.logTrace(`${dat.getNameShort()}: no input ROMs to make candidates from`);
return new Map();
}
Expand Down Expand Up @@ -71,7 +72,7 @@ export default class CandidateGenerator extends Module {
dat,
game,
release,
hashCodeToInputFiles,
indexedFiles,
);
if (releaseCandidate) {
releaseCandidates.push(releaseCandidate);
Expand Down Expand Up @@ -104,9 +105,9 @@ export default class CandidateGenerator extends Module {
dat: DAT,
game: Game,
release: Release | undefined,
hashCodeToInputFiles: Map<string, File[]>,
indexedFiles: IndexedFiles,
): Promise<ReleaseCandidate | undefined> {
const romsToInputFiles = CandidateGenerator.getInputFilesForGame(game, hashCodeToInputFiles);
const romsToInputFiles = this.getInputFilesForGame(game, indexedFiles);

// For each Game's ROM, find the matching File
const romFiles = await Promise.all(
Expand Down Expand Up @@ -136,9 +137,12 @@ export default class CandidateGenerator extends Module {
) {
// No automatic header removal will be performed when raw-copying an archive, so return no
// match if we wanted a headerless ROM but got a headered one.
if (rom.hashCode() !== originalInputFile.hashCodeWithHeader()
&& rom.hashCode() === originalInputFile.hashCodeWithoutHeader()
if (originalInputFile.getFileHeader()
&& !(rom.getCrc32() === originalInputFile.getCrc32()
|| rom.getMd5() === originalInputFile.getMd5()
|| rom.getSha1() === originalInputFile.getSha1())
) {
// TODO(cemmer): is this right?
return [rom, undefined];
}

Expand Down Expand Up @@ -189,13 +193,13 @@ export default class CandidateGenerator extends Module {
return new ReleaseCandidate(game, release, foundRomsWithFiles);
}

private static getInputFilesForGame(
private getInputFilesForGame(

Check failure on line 196 in src/modules/candidateGenerator.ts

View workflow job for this annotation

GitHub Actions / node-lint 

Expected 'this' to be used by class method 'getInputFilesForGame'
game: Game,
hashCodeToInputFiles: Map<string, File[]>,
indexedFiles: IndexedFiles,
): Map<ROM, File> {
let romsAndInputFiles = game.getRoms().map((rom) => ([
rom,
(hashCodeToInputFiles.get(rom.hashCode()) ?? []),
indexedFiles.findFiles(rom) ?? [],
])) satisfies [ROM, File[]][];

// Detect if there is one input archive that contains every ROM, and prefer to use its entries.
Expand Down
104 changes: 41 additions & 63 deletions src/modules/fileIndexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import SevenZip from '../types/files/archives/sevenZip.js';
import Tar from '../types/files/archives/tar.js';
import Zip from '../types/files/archives/zip.js';
import File from '../types/files/file.js';
import IndexedFiles, { AllChecksums, ChecksumsToFiles } from '../types/indexedFiles.js';
import Options from '../types/options.js';
import Module from './module.js';

Expand All @@ -26,83 +27,60 @@ export default class FileIndexer extends Module {
/**
* Index files.
*/
async index(files: File[]): Promise<Map<string, File[]>> {
if (files.length === 0) {
return new Map();
}

async index(files: File[]): Promise<IndexedFiles> {
this.progressBar.logTrace(`indexing ${files.length.toLocaleString()} file${files.length !== 1 ? 's' : ''}`);
await this.progressBar.setSymbol(ProgressBarSymbol.INDEXING);
// await this.progressBar.reset(files.length);

const results = new Map<string, File[]>();
// Index the files
const result = IndexedFiles.fromFiles(files);
// Then apply some sorting preferences
Object.keys(result).forEach((checksum) => this.sortMap(result[checksum as keyof AllChecksums]));

// TODO(cemmer): ability to index files by some other property such as name
files.forEach((file) => {
// Index on full file contents
FileIndexer.setFileInMap(results, file.hashCodeWithHeader(), file);
this.progressBar.logTrace(`found ${result.getSize()} unique file${result.getSize() !== 1 ? 's' : ''}`);

// Optionally index without a header
if (file.getFileHeader()) {
FileIndexer.setFileInMap(results, file.hashCodeWithoutHeader(), file);
}
});
this.progressBar.logTrace('done indexing files');
return result;
}

private sortMap(checksumsToFiles: ChecksumsToFiles): void {
const outputDir = path.resolve(this.options.getOutputDirRoot());
const outputDirDisk = FsPoly.disksSync().find((mount) => outputDir.startsWith(mount));

// Sort the file arrays
[...results.entries()]
.forEach(([hashCode, filesForHash]) => filesForHash.sort((fileOne, fileTwo) => {
// First, prefer "raw" files (files with their header)
const fileOneHeadered = fileOne.getFileHeader()
&& fileOne.hashCodeWithoutHeader() === hashCode ? 1 : 0;
const fileTwoHeadered = fileTwo.getFileHeader()
&& fileTwo.hashCodeWithoutHeader() === hashCode ? 1 : 0;
if (fileOneHeadered !== fileTwoHeadered) {
return fileOneHeadered - fileTwoHeadered;
}

// Then, prefer un-archived files
const fileOneArchived = FileIndexer.archiveEntryPriority(fileOne);
const fileTwoArchived = FileIndexer.archiveEntryPriority(fileTwo);
if (fileOneArchived !== fileTwoArchived) {
return fileOneArchived - fileTwoArchived;
}

// Then, prefer files that are NOT already in the output directory
// This is in case the output file is invalid and we're trying to overwrite it with
// something else. Otherwise, we'll just attempt to overwrite the invalid output file with
// itself, still resulting in an invalid output file.
const fileOneInOutput = path.resolve(fileOne.getFilePath()).startsWith(outputDir) ? 1 : 0;
const fileTwoInOutput = path.resolve(fileTwo.getFilePath()).startsWith(outputDir) ? 1 : 0;
if (fileOneInOutput !== fileTwoInOutput) {
return fileOneInOutput - fileTwoInOutput;
}

// Then, prefer files that are on the same disk for fs efficiency see {@link FsPoly#mv}
if (outputDirDisk) {
const fileOneInOutputDisk = path.resolve(fileOne.getFilePath())
.startsWith(outputDirDisk) ? 0 : 1;
const fileTwoInOutputDisk = path.resolve(fileTwo.getFilePath())
.startsWith(outputDirDisk) ? 0 : 1;
if (fileOneInOutputDisk !== fileTwoInOutputDisk) {
return fileOneInOutputDisk - fileTwoInOutputDisk;
[...checksumsToFiles.values()]
.forEach((files) => files
.sort((fileOne, fileTwo) => {
// Prefer un-archived files
const fileOneArchived = FileIndexer.archiveEntryPriority(fileOne);
const fileTwoArchived = FileIndexer.archiveEntryPriority(fileTwo);
if (fileOneArchived !== fileTwoArchived) {
return fileOneArchived - fileTwoArchived;
}
}

// Otherwise, be deterministic
return fileOne.getFilePath().localeCompare(fileTwo.getFilePath());
}));

this.progressBar.logTrace(`found ${results.size} unique file${results.size !== 1 ? 's' : ''}`);
// Then, prefer files that are NOT already in the output directory
// This is in case the output file is invalid and we're trying to overwrite it with
// something else. Otherwise, we'll just attempt to overwrite the invalid output file with
// itself, still resulting in an invalid output file.
const fileOneInOutput = path.resolve(fileOne.getFilePath()).startsWith(outputDir) ? 1 : 0;
const fileTwoInOutput = path.resolve(fileTwo.getFilePath()).startsWith(outputDir) ? 1 : 0;
if (fileOneInOutput !== fileTwoInOutput) {
return fileOneInOutput - fileTwoInOutput;
}

this.progressBar.logTrace('done indexing files');
return results;
}
// Then, prefer files that are on the same disk for fs efficiency see {@link FsPoly#mv}
if (outputDirDisk) {
const fileOneInOutputDisk = path.resolve(fileOne.getFilePath())
.startsWith(outputDirDisk) ? 0 : 1;
const fileTwoInOutputDisk = path.resolve(fileTwo.getFilePath())
.startsWith(outputDirDisk) ? 0 : 1;
if (fileOneInOutputDisk !== fileTwoInOutputDisk) {
return fileOneInOutputDisk - fileTwoInOutputDisk;
}
}

private static setFileInMap<K>(map: Map<K, File[]>, key: K, file: File): void {
map.set(key, [...(map.get(key) ?? []), file]);
// Otherwise, be deterministic
return fileOne.getFilePath().localeCompare(fileTwo.getFilePath());
}));
}

/**
Expand Down
4 changes: 2 additions & 2 deletions src/modules/movedRomDeleter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ export default class MovedROMDeleter extends Module {
// the unique set of ArchiveEntry hash codes to know if every ArchiveEntry was "consumed"
// during writing.
const movedEntryHashCodes = new Set(
movedEntries.flatMap((file) => file.hashCodes()),
movedEntries.flatMap((file) => file.hashCode()),
);

const inputEntries = groupedInputRoms.get(filePath) ?? [];
Expand All @@ -94,7 +94,7 @@ export default class MovedROMDeleter extends Module {
}

// Otherwise, the entry needs to have been explicitly moved
return entry.hashCodes().some((hashCode) => !movedEntryHashCodes.has(hashCode));
return !movedEntryHashCodes.has(entry.hashCode());
});
if (unmovedEntries.length > 0) {
this.progressBar.logWarn(`${filePath}: not deleting moved file, ${unmovedEntries.length.toLocaleString()} archive entr${unmovedEntries.length !== 1 ? 'ies were' : 'y was'} unmatched:\n${unmovedEntries.sort().map((entry) => ` ${entry}`).join('\n')}`);
Expand Down
2 changes: 1 addition & 1 deletion src/modules/scanner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export default abstract class Scanner extends Module {
): Promise<File[]> {
const foundFiles = await this.getFilesFromPaths(filePaths, threads, checksumBitmask);
return foundFiles
.filter(ArrayPoly.filterUniqueMapped((file) => file.hashCodes().join(',')));
.filter(ArrayPoly.filterUniqueMapped((file) => file.hashCode()));
}

private async getFilesFromPath(
Expand Down
4 changes: 3 additions & 1 deletion src/types/dats/rom.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ export default class ROM implements ROMProps {
* A string hash code to uniquely identify this {@link ROM}.
*/
hashCode(): string {
return File.hashCode(this.getCrc32(), this.getSize());
return this.getSha1()
?? this.getMd5()
?? `${this.getCrc32()}|${this.getSize()}`;
}
}
25 changes: 8 additions & 17 deletions src/types/files/file.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,7 @@ export default class File implements FileProps {
****************************
*/

// TODO(cemmer): refactor usages of this that should use hashCode() or something else
toString(): string {
// TODO(cemmer): indicate if there's a patch?
if (this.getSymlinkSource()) {
Expand All @@ -445,23 +446,13 @@ export default class File implements FileProps {
return this.getFilePath();
}

static hashCode(crc: string, size: number): string {
return `${crc}|${size}`;
}

hashCodeWithHeader(): string {
return File.hashCode(this.getCrc32(), this.getSize());
}

hashCodeWithoutHeader(): string {
return File.hashCode(this.getCrc32WithoutHeader(), this.getSizeWithoutHeader());
}

hashCodes(): string[] {
return [
this.hashCodeWithHeader(),
this.hashCodeWithoutHeader(),
].reduce(ArrayPoly.reduceUnique(), []);
/**
* A string hash code to uniquely identify this {@link File}.
*/
hashCode(): string {
return this.getSha1()
?? this.getMd5()
?? `${this.getCrc32()}|${this.getSize()}`;
}

equals(other: File): boolean {
Expand Down
Loading

0 comments on commit 2f92d86

Please sign in to comment.