Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor: explicitly flush written files to disk #1094

Merged
merged 3 commits into from
Apr 12, 2024
Merged

Refactor: explicitly flush written files to disk #1094

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd26bf0
Refactor: explicitly flush written files to disk
emmercm Apr 12, 2024
c1f125c
Merge branch 'main' into emmercm/flush-written-files
emmercm Apr 12, 2024
89517ec
Merge branch 'main' into emmercm/flush-written-files
emmercm Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions src/modules/dir2DatCreator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import fs from 'node:fs';
import path from 'node:path';

import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js';
Expand Down Expand Up @@ -41,7 +40,7 @@ export default class Dir2DatCreator extends Module {

this.progressBar.logInfo(`${dat.getNameShort()}: creating dir2dat '${datPath}'`);
const datContents = dat.toXmlDat();
await fs.promises.writeFile(datPath, datContents);
await fsPoly.writeFile(datPath, datContents);

this.progressBar.logTrace(`${dat.getNameShort()}: done writing dir2dat`);
return datPath;
Expand Down
3 changes: 1 addition & 2 deletions src/modules/fixdatCreator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import fs from 'node:fs';
import path from 'node:path';

import moment from 'moment';
Expand Down Expand Up @@ -84,7 +83,7 @@ export default class FixdatCreator extends Module {
const fixdatContents = fixdat.toXmlDat();
const fixdatPath = path.join(fixdatDir, fixdat.getFilename());
this.progressBar.logInfo(`${originalDat.getNameShort()}: writing fixdat to '${fixdatPath}'`);
await fs.promises.writeFile(fixdatPath, fixdatContents);
await fsPoly.writeFile(fixdatPath, fixdatContents);

this.progressBar.logTrace(`${originalDat.getNameShort()}: done generating a fixdat`);
return fixdatPath;
Expand Down
5 changes: 2 additions & 3 deletions src/modules/reportGenerator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import fs from 'node:fs';

import ProgressBar from '../console/progressBar.js';
import ArrayPoly from '../polyfill/arrayPoly.js';
import FsPoly from '../polyfill/fsPoly.js';
import DATStatus, { GameStatus } from '../types/datStatus.js';
import Options from '../types/options.js';
import Module from './module.js';
Expand Down Expand Up @@ -59,7 +58,7 @@ export default class ReportGenerator extends Module {

this.progressBar.logInfo(`writing report '${reportPath}'`);
const rows = [...matchedFileCsvs, unusedCsv, cleanedCsv].filter((csv) => csv);
await fs.promises.writeFile(reportPath, rows.join('\n'));
await FsPoly.writeFile(reportPath, rows.join('\n'));
this.progressBar.logTrace(`wrote ${datStatuses.length.toLocaleString()} CSV row${datStatuses.length !== 1 ? 's' : ''}: ${reportPath}`);

this.progressBar.logTrace('done generating report');
Expand Down
15 changes: 14 additions & 1 deletion src/polyfill/fsPoly.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import crypto from 'node:crypto';
import fs, { MakeDirectoryOptions, PathLike, RmOptions } from 'node:fs';
import fs, {
MakeDirectoryOptions, ObjectEncodingOptions, PathLike, RmOptions,
} from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import util from 'node:util';
Expand Down Expand Up @@ -430,4 +432,15 @@ export default class FsPoly {

return output;
}

static async writeFile(
filePath: PathLike,
data: string | NodeJS.ArrayBufferView,
options?: ObjectEncodingOptions,
): Promise<void> {
const file = await fs.promises.open(filePath, 'w');
await fs.promises.writeFile(file, data, options);
await file.sync(); // emulate fs.promises.writeFile() flush:true added in v21.0.0
await file.close();
}
}
2 changes: 1 addition & 1 deletion src/types/cache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ export default class Cache<V> {

// Write to a temp file first, then overwrite the old cache file
const tempFile = await FsPoly.mktemp(this.filePath);
await fs.promises.writeFile(
await FsPoly.writeFile(
tempFile,
JSON.stringify(cacheData),
{ encoding: Cache.BUFFER_ENCODING },
Expand Down
17 changes: 8 additions & 9 deletions test/modules/argumentsParser.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';

Expand Down Expand Up @@ -270,7 +269,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-name-regex', tempFile]).getDatNameRegex()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-name-regex', tempFile]).getDatNameRegex()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-name-regex', tempFile]).getDatNameRegex()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand All @@ -295,7 +294,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-name-regex-exclude', tempFile]).getDatNameRegexExclude()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-name-regex-exclude', tempFile]).getDatNameRegexExclude()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-name-regex-exclude', tempFile]).getDatNameRegexExclude()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand All @@ -316,7 +315,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-description-regex', tempFile]).getDatDescriptionRegex()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-description-regex', tempFile]).getDatDescriptionRegex()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-description-regex', tempFile]).getDatDescriptionRegex()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand All @@ -337,7 +336,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-description-regex-exclude', tempFile]).getDatDescriptionRegexExclude()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-description-regex-exclude', tempFile]).getDatDescriptionRegexExclude()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--dat-description-regex-exclude', tempFile]).getDatDescriptionRegexExclude()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand Down Expand Up @@ -622,7 +621,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--single', '--prefer-game-regex', tempFile]).getPreferGameRegex()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--single', '--prefer-game-regex', tempFile]).getPreferGameRegex()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--single', '--prefer-game-regex', tempFile]).getPreferGameRegex()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand All @@ -642,7 +641,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--single', '--prefer-rom-regex', tempFile]).getPreferRomRegex()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--single', '--prefer-rom-regex', tempFile]).getPreferRomRegex()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--single', '--prefer-rom-regex', tempFile]).getPreferRomRegex()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand Down Expand Up @@ -786,7 +785,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--filter-regex', tempFile]).getFilterRegex()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--filter-regex', tempFile]).getFilterRegex()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--filter-regex', tempFile]).getFilterRegex()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand All @@ -807,7 +806,7 @@ describe('options', () => {

const tempFile = await FsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'temp'));
try {
await fs.promises.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
await FsPoly.writeFile(tempFile, '\n/[a-z]/i\r\n[0-9]\n\n');
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--filter-regex-exclude', tempFile]).getFilterRegexExclude()?.some((regex) => regex.test(''))).toEqual(false);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--filter-regex-exclude', tempFile]).getFilterRegexExclude()?.some((regex) => regex.test('lower'))).toEqual(true);
expect(argumentsParser.parse([...dummyCommandAndRequiredArgs, '--filter-regex-exclude', tempFile]).getFilterRegexExclude()?.some((regex) => regex.test('UPPER'))).toEqual(true);
Expand Down
6 changes: 3 additions & 3 deletions test/polyfill/filePoly.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ describe('getSize', () => {
describe('readNext', () => {
it('should read from the beginning', async () => {
const tempFile = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'file'));
await fs.promises.writeFile(tempFile, 'ABCDEF0123456789');
await fsPoly.writeFile(tempFile, 'ABCDEF0123456789');

const file = await filePoly.fileFrom(tempFile, 'r');
try {
Expand All @@ -105,7 +105,7 @@ describe('readNext', () => {

it('should respect seek', async () => {
const tempFile = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'file'));
await fs.promises.writeFile(tempFile, 'ABCDEF0123456789');
await fsPoly.writeFile(tempFile, 'ABCDEF0123456789');

const file = await filePoly.fileFrom(tempFile, 'r');
try {
Expand All @@ -121,7 +121,7 @@ describe('readNext', () => {

it('should respect skipNext', async () => {
const tempFile = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, 'file'));
await fs.promises.writeFile(tempFile, 'ABCDEF0123456789');
await fsPoly.writeFile(tempFile, 'ABCDEF0123456789');

const file = await filePoly.fileFrom(tempFile, 'r');
try {
Expand Down
2 changes: 1 addition & 1 deletion test/types/patches/apsGbaPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import APSPatch from '../../../src/types/patches/apsPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
2 changes: 1 addition & 1 deletion test/types/patches/apsN64Patch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import APSPatch from '../../../src/types/patches/apsPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
2 changes: 1 addition & 1 deletion test/types/patches/bpsPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import BPSPatch from '../../../src/types/patches/bpsPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
3 changes: 1 addition & 2 deletions test/types/patches/dpsPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import fs from 'node:fs';
import path from 'node:path';

import Constants from '../../../src/constants.js';
Expand All @@ -10,7 +9,7 @@ import DPSPatch from '../../../src/types/patches/dpsPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
Binary file modified test/types/patches/ipsPatch.test.ts
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion test/types/patches/ninjaPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import NinjaPatch from '../../../src/types/patches/ninjaPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
2 changes: 1 addition & 1 deletion test/types/patches/ppfPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import PPFPatch from '../../../src/types/patches/ppfPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
2 changes: 1 addition & 1 deletion test/types/patches/upsPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import UPSPatch from '../../../src/types/patches/upsPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
2 changes: 1 addition & 1 deletion test/types/patches/vcdiffPatch.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import VcdiffPatch from '../../../src/types/patches/vcdiffPatch.js';

async function writeTemp(fileName: string, contents: string | Buffer): Promise<File> {
const temp = await fsPoly.mktemp(path.join(Constants.GLOBAL_TEMP_DIR, fileName));
await fs.promises.writeFile(temp, contents);
await fsPoly.writeFile(temp, contents);
return File.fileOf({ filePath: temp });
}

Expand Down
Loading