diff --git a/README.md b/README.md index 1dcea7eb..ab80f0cd 100644 --- a/README.md +++ b/README.md @@ -17,5 +17,6 @@ Here is a brief overview of the included libraries: 11. [`exit-hook`](./packages/exit-hook/README.md): A utility for registering exit handlers in Node.js. 12. [`flatomise`](./packages/flatomise/README.md): A utility for creating promises that can be externally resolved or rejected. 13. [`async-queue`](./packages/async-queue/README.md): A queue that executes async tasks in order like mutex and semaphore methodology for javascript and typescript. +14. [`node-fs`](./packages/node-fs/README.md): Enhanced file system operations in Node.js, including reading, writing, and handling JSON files, with both synchronous and asynchronous options. -For more detailed information and guidelines on how to use each package, please refer to to each package's README. +For more detailed information and guidelines on how to use each package, please refer to each package's README. diff --git a/packages/logger/src/define-package.ts b/packages/logger/src/define-package.ts index abaa169b..20d22868 100644 --- a/packages/logger/src/define-package.ts +++ b/packages/logger/src/define-package.ts @@ -1,8 +1,8 @@ import {definePackage as definePackage_} from '@alwatr/dedupe'; -import {createLogger} from './logger.js'; +import {createLogger} from './logger'; -import type {AlwatrLogger} from './type.js'; +import type {AlwatrLogger} from './type'; /** * Global define package for managing package versions to prevent version conflicts and return package level logger. diff --git a/packages/logger/src/logger.ts b/packages/logger/src/logger.ts index 162757e9..ae59a915 100644 --- a/packages/logger/src/logger.ts +++ b/packages/logger/src/logger.ts @@ -1,7 +1,7 @@ import {definePackage} from '@alwatr/dedupe'; import {platformInfo} from '@alwatr/platform-info'; -import type {AlwatrLogger} from './type.js'; +import type {AlwatrLogger} from './type'; definePackage('@alwatr/logger', __package_version__); diff --git a/packages/node-fs/README.md b/packages/node-fs/README.md new file mode 100644 index 00000000..f7c96997 --- /dev/null +++ b/packages/node-fs/README.md @@ -0,0 +1,37 @@ +# Node FS + +Enhanced file system operations in Node.js with asynchronous queue to prevent parallel writes. + +## Installation + +```bash +yarn add @alwatr/node-fs +``` + +## Features + +- Checks if a directory exists. If it doesn't, it creates the directory and all necessary subdirectories. +- Before writing a file successfully, first writes it to a temporary path (`path.tmp`). +- If a file already exists, renames and keeps the existing file at a backup path (`path.bak`). +- If a write operation fails, the original file remains unchanged. +- Includes `readJson` and `writeJson` functions that automatically parse and stringify JSON data. +- Supports both synchronous and asynchronous read/write operations. +- An asynchronous queue is used to prevent simultaneous write operations. +- Fully written in TypeScript, includes type definitions. +- Separate builds are provided for ESModule and CommonJS. +- Zero dependencies, except for the nanolib library. +- Includes a beautiful log feature, which uses the [logger](https://github.com/Alwatr/nanolib/tree/next/packages/logger) package from nanolib. + +## Usage + +```typescript +import {writeJson} from '@alwatr/node-fs'; + +const path = 'file.json'; +await writeJson(path, {a: 1}); // wait to finish +writeJson(path, {a: 2}); // asynchronous write in queue +writeJson(path, {a: 3}); // asynchronous write in queue + +const data = await readJson(path); // automatically wait for the queue to finish +console.log(data.a); // 3 +``` diff --git a/packages/node-fs/demo/make-file-bench.mjs b/packages/node-fs/demo/make-file-bench.mjs new file mode 100644 index 00000000..5cad805b --- /dev/null +++ b/packages/node-fs/demo/make-file-bench.mjs @@ -0,0 +1,20 @@ +import {existsSync, makeEmptyFile, resolve} from '@alwatr/node-fs'; + +import {mkdir, rm} from 'node:fs/promises'; + +(async () => { + const temp = resolve('./.tmp'); + + await mkdir(temp); + + console.log('start bench'); + + console.time('bench'); + for (let i = 0; i < 10_000; i++) { + await makeEmptyFile(`${temp}/file-${i}.asn`); + } + console.timeEnd('bench'); + + await rm(temp, {recursive: true, force: true}); + +})(); diff --git a/packages/node-fs/demo/node-fs.mjs b/packages/node-fs/demo/node-fs.mjs new file mode 100644 index 00000000..566fc3d5 --- /dev/null +++ b/packages/node-fs/demo/node-fs.mjs @@ -0,0 +1,14 @@ +import {readJson, writeJson} from '@alwatr/node-fs'; + +for (let i = 0; i < 100; i++) { + console.log('writeJson %s without waiting...', i); + writeJson('file.json', {i, a: 'b'}); + if (i===70) { + console.log('readJson %s', i); + console.dir(await readJson('file.json')); + } +} + +console.dir(await readJson('file.json')); + +console.log('loop done, wait for queue process') diff --git a/packages/node-fs/package.json b/packages/node-fs/package.json new file mode 100644 index 00000000..4077c08b --- /dev/null +++ b/packages/node-fs/package.json @@ -0,0 +1,88 @@ +{ + "name": "@alwatr/node-fs", + "version": "0.0.0", + "description": "Enhanced file system operations in Node.js with asynchronous queue to prevent parallel writes.", + "author": "S. Ali Mihandoost ", + "keywords": [ + "node-fs", + "fs", + "file", + "filesystem", + "readFile", + "writeFile", + "readJson", + "writeJson", + "JSON", + "async", + "queue", + "cross-platform", + "ECMAScript", + "typescript", + "javascript", + "node", + "nodejs", + "esm", + "module", + "utility", + "util", + "utils", + "nanolib", + "alwatr" + ], + "type": "module", + "main": "./dist/main.cjs", + "module": "./dist/main.mjs", + "types": "./dist/main.d.ts", + "exports": { + ".": { + "import": "./dist/main.mjs", + "require": "./dist/main.cjs", + "types": "./dist/main.d.ts" + } + }, + "license": "MIT", + "files": [ + "**/*.{js,mjs,cjs,map,d.ts,html,md}", + "!demo/**/*" + ], + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/Alwatr/nanolib", + "directory": "packages/node-fs" + }, + "homepage": "https://github.com/Alwatr/nanolib/tree/next/packages/node-fs#readme", + "bugs": { + "url": "https://github.com/Alwatr/nanolib/issues" + }, + "prettier": "@alwatr/prettier-config", + "scripts": { + "b": "yarn run build", + "w": "yarn run watch", + "c": "yarn run clean", + "cb": "yarn run clean && yarn run build", + "d": "yarn run build:es && yarn node --enable-source-maps --trace-warnings", + "build": "yarn run build:ts & yarn run build:es", + "build:es": "nano-build --preset=module", + "build:ts": "tsc --build", + "watch": "yarn run watch:ts & yarn run watch:es", + "watch:es": "yarn run build:es --watch", + "watch:ts": "yarn run build:ts --watch --preserveWatchOutput", + "clean": "rm -rfv dist *.tsbuildinfo" + }, + "dependencies": { + "@alwatr/async-queue": "workspace:^", + "@alwatr/flat-string": "workspace:^", + "@alwatr/logger": "workspace:^" + }, + "devDependencies": { + "@alwatr/nano-build": "workspace:^", + "@alwatr/prettier-config": "workspace:^", + "@alwatr/tsconfig-base": "workspace:^", + "@alwatr/type-helper": "workspace:^", + "@types/node": "^20.10.7", + "typescript": "^5.3.3" + } +} diff --git a/packages/node-fs/src/common.ts b/packages/node-fs/src/common.ts new file mode 100644 index 00000000..45a80550 --- /dev/null +++ b/packages/node-fs/src/common.ts @@ -0,0 +1,8 @@ +import {AsyncQueue} from '@alwatr/async-queue'; +import {definePackage} from '@alwatr/logger'; + +import type {} from '@alwatr/nano-build'; + +export const logger = definePackage('@alwatr/node-fs', __package_version__); + +export const asyncQueue = new AsyncQueue(); diff --git a/packages/node-fs/src/json.ts b/packages/node-fs/src/json.ts new file mode 100644 index 00000000..3b9c4802 --- /dev/null +++ b/packages/node-fs/src/json.ts @@ -0,0 +1,45 @@ +import {logger} from './common'; + +import type { Dictionary } from '@alwatr/type-helper'; + +/** + * Parse json string. + * + * @param content - json string + * @returns json object + * @example + * ```typescript + * const json = parseJson('{"a":1,"b":2}'); + * console.log(json.a); // 1 + * ``` + */ +export function parseJson(content: string): T { + try { + return JSON.parse(content); + } + catch (err) { + logger.error('parseJson', 'invalid_json', err); + throw new Error('invalid_json', {cause: (err as Error).cause}); + } +} + +/** + * Stringify json object. + * + * @param data - json object + * @returns json string + * @example + * ```typescript + * const json = jsonStringify({a:1, b:2}); + * console.log(json); // '{"a":1,"b":2}' + * ``` + */ +export function jsonStringify(data: T): string { + try { + return JSON.stringify(data); + } + catch (err) { + logger.error('jsonStringify', 'stringify_failed', err); + throw new Error('stringify_failed', {cause: (err as Error).cause}); + } +} diff --git a/packages/node-fs/src/main.ts b/packages/node-fs/src/main.ts new file mode 100644 index 00000000..13ab59ad --- /dev/null +++ b/packages/node-fs/src/main.ts @@ -0,0 +1,9 @@ +export * from './read-file'; +export * from './write-file'; +export * from './read-json'; +export * from './write-json'; +export * from './make-file'; + +export {resolve} from 'node:path'; +export {existsSync} from 'node:fs'; +export {unlink} from 'node:fs/promises'; diff --git a/packages/node-fs/src/make-file.ts b/packages/node-fs/src/make-file.ts new file mode 100644 index 00000000..8b9649af --- /dev/null +++ b/packages/node-fs/src/make-file.ts @@ -0,0 +1,18 @@ +import {open} from 'node:fs/promises'; + +import {logger} from './common'; + +/** + * Make empty file. + * + * @param path - file path + * + * @example + * ```ts + * await makeFile('./file.txt'); + * ``` + */ +export async function makeEmptyFile(path: string): Promise { + logger.logMethodArgs?.('makeEmptyFile', '...' + path.slice(-32)); + return (await open(path, 'w')).close(); +} diff --git a/packages/node-fs/src/read-file.ts b/packages/node-fs/src/read-file.ts new file mode 100644 index 00000000..b00f84c2 --- /dev/null +++ b/packages/node-fs/src/read-file.ts @@ -0,0 +1,54 @@ +import {readFileSync as readFileSync_} from 'node:fs'; +import {readFile as readFile_} from 'node:fs/promises'; + +import {flatString} from '@alwatr/flat-string'; + +import {asyncQueue, logger} from './common'; + +/** + * Enhanced read File (Synchronous). + * + * @param path - file path + * @returns file content + * @example + * ```typescript + * const fileContent = readFileSync('./file.txt', sync); + * ``` + */ +export function readFileSync(path: string): string { + logger.logMethodArgs?.('readFileSync', '...' + path.slice(-32)); + // if (!existsSync(path)) throw new Error('file_not_found'); + try { + return flatString(readFileSync_(path, {encoding: 'utf-8', flag: 'r'})); + } + catch (err) { + logger.error('readFileSync', 'read_file_failed', {path}, err); + throw new Error('read_file_failed', {cause: (err as Error).cause}); + } +} + +/** + * Enhanced read File (Asynchronous). + * + * - If writing queue is running for target path, it will wait for it to finish. + * + * @param path - file path + * @returns file content + * @example + * ```typescript + * const fileContent = await readFile('./file.txt', sync); + * ``` + */ +export function readFile(path: string): Promise { + logger.logMethodArgs?.('readFile', '...' + path.slice(-32)); + // if (!existsSync(path)) throw new Error('file_not_found'); + return asyncQueue.push(path, async () => { + try { + return flatString(await readFile_(path, {encoding: 'utf-8', flag: 'r'})); + } + catch (err) { + logger.error('readFile', 'read_file_failed', {path}, err); + throw new Error('read_file_failed', {cause: (err as Error).cause}); + } + }); +} diff --git a/packages/node-fs/src/read-json.ts b/packages/node-fs/src/read-json.ts new file mode 100644 index 00000000..84321e8e --- /dev/null +++ b/packages/node-fs/src/read-json.ts @@ -0,0 +1,61 @@ +import {logger} from './common'; +import {parseJson} from './json'; +import {readFile, readFileSync} from './read-file'; + +import type {Dictionary, MaybePromise} from '@alwatr/type-helper'; + +/** + * Enhanced read json file (async). + * + * @param path - file path + * @returns json object + * @example + * ```typescript + * const fileContent = await readJson('./file.json'); + * ``` + */ +export function readJson(path: string): Promise; +/** + * Enhanced read json file (sync). + * + * @param path - file path + * @param sync - sync mode + * @returns json object + * @example + * ```typescript + * const fileContent = readJson('./file.json', true); + * ``` + */ +export function readJson(path: string, sync: true): T; +/** + * Enhanced read json file. + * + * @param path - file path + * @param sync - sync mode + * @returns json object + * @example + * ```typescript + * const fileContent = await readJson('./file.json', sync); + * ``` + */ +export function readJson(path: string, sync: boolean): MaybePromise; +/** + * Enhanced read json file. + * + * @param path - file path + * @param sync - sync mode + * @returns json object + * @example + * ```typescript + * const fileContent = await readJson('./file.json'); + * ``` + */ +export function readJson(path: string, sync = false): MaybePromise { + logger.logMethodArgs?.('readJson', {path: path.slice(-32), sync}); + if (sync === true) { + return parseJson(readFileSync(path)); + } + else { + return readFile(path).then((content) => parseJson(content)); + } +} diff --git a/packages/node-fs/src/write-file.ts b/packages/node-fs/src/write-file.ts new file mode 100644 index 00000000..baf59f78 --- /dev/null +++ b/packages/node-fs/src/write-file.ts @@ -0,0 +1,84 @@ +import {writeFileSync as writeFileSync_, existsSync, mkdirSync, renameSync} from 'node:fs'; +import {mkdir, rename, writeFile as writeFile_} from 'node:fs/promises'; +import {dirname} from 'node:path'; + +import {asyncQueue, logger} from './common'; + +/** + * Enhanced write file (Synchronous). + * + * - If directory not exists, create it recursively. + * - Write file to `path.tmp` before write success. + * - If file exists, renamed (keep) to `path.bak`. + * - If write failed, original file will not be changed. + * + * @param path - file path + * @param content - file content + * @example + * ```typescript + * writeFileSync('./file.txt', 'Hello World!'); + * ``` + */ +export function writeFileSync(path: string, content: string): void { + logger.logMethodArgs?.('writeFileSync', '...' + path.slice(-32)); + try { + const pathExists = existsSync(path); + if (!pathExists) { + const dir = dirname(path); + if (!existsSync(dir)) { + mkdirSync(dir, {recursive: true}); + } + } + writeFileSync_(path + '.tmp', content, {encoding: 'utf-8', flag: 'w'}); + if (pathExists) { + renameSync(path, path + '.bak'); + } + renameSync(path + '.tmp', path); + logger.logOther?.('writeFileSync success', '...' + path.slice(-32)); + } + catch (err) { + logger.error('writeFileSync', 'write_file_failed', {path}, err); + throw new Error('write_file_failed', {cause: (err as Error).cause}); + } +} + +/** + * Enhanced write file (Asynchronous). + * + * - If directory not exists, create it recursively. + * - Write file to `path.tmp` before write success. + * - If file exists, renamed (keep) to `path.bak`. + * - If write failed, original file will not be changed. + * + * @param path - file path + * @param content - file content + * @example + * ```typescript + * await writeFile('./file.txt', 'Hello World!'); + * ``` + */ +export function writeFile(path: string, content: string): Promise { + logger.logMethodArgs?.('writeFile', '...' + path.slice(-32)); + return asyncQueue.push(path, async () => { + try { + logger.logOther?.('writeFile start', '...' + path.slice(-32)); + const pathExists = existsSync(path); + if (!pathExists) { + const dir = dirname(path); + if (!existsSync(dir)) { + await mkdir(dir, {recursive: true}); + } + } + await writeFile_(path + '.tmp', content, {encoding: 'utf-8', flag: 'w'}); + if (pathExists) { + await rename(path, path + '.bak'); + } + await rename(path + '.tmp', path); + logger.logOther?.('writeFile success', '...' + path.slice(-32)); + } + catch (err) { + logger.error('writeFile', 'write_file_failed', {path}, err); + throw new Error('write_file_failed', {cause: (err as Error).cause}); + } + }); +} diff --git a/packages/node-fs/src/write-json.ts b/packages/node-fs/src/write-json.ts new file mode 100644 index 00000000..897aafab --- /dev/null +++ b/packages/node-fs/src/write-json.ts @@ -0,0 +1,59 @@ +import {flatString} from '@alwatr/flat-string'; + +import {logger} from './common'; +import {jsonStringify} from './json'; +import {writeFile, writeFileSync} from './write-file'; + +import type {Dictionary, MaybePromise} from '@alwatr/type-helper'; + +/** + * Enhanced write json file (Asynchronous). + * + * @param path - file path + * @param data - json object + * @example + * ```typescript + * await writeJsonFile('./file.json', { a:1, b:2, c:3 }); + * ``` + */ +export function writeJson(path: string, data: T, sync?: false): Promise; +/** + * Enhanced write json file (Synchronous). + * + * @param path - file path + * @param data - json object + * @param sync - sync mode + * @example + * ```typescript + * writeJsonFile('./file.json', { a:1, b:2, c:3 }, true); + * ``` + */ +export function writeJson(path: string, data: T, sync: true): void; +/** + * Enhanced write json file. + * + * @param path - file path + * @param data - json object + * @param sync - sync mode + * @example + * ```typescript + * await writeJsonFile('./file.json', { a:1, b:2, c:3 }, sync); + * ``` + */ +export function writeJson(path: string, data: T, sync: boolean): MaybePromise; +/** + * Enhanced write json file. + * + * @param path - file path + * @param data - json object + * @param sync - sync mode + * @example + * ```typescript + * await writeJsonFile('./file.json', { a:1, b:2, c:3 }); + * ``` + */ +export function writeJson(path: string, data: T, sync = false): MaybePromise { + logger.logMethodArgs?.('writeJson', '...' + path.slice(-32)); + const content = flatString(jsonStringify(data)); + return sync === true ? writeFileSync(path, content) : writeFile(path, content); +} diff --git a/packages/node-fs/tsconfig.json b/packages/node-fs/tsconfig.json new file mode 100644 index 00000000..34c764f4 --- /dev/null +++ b/packages/node-fs/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "@alwatr/tsconfig-base/tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "emitDeclarationOnly": true, + "composite": true, + }, + "include": ["src/**/*.ts"], + "references": [ + {"path": "../async-queue"}, + {"path": "../flat-string"}, + {"path": "../logger"}, + ] +} diff --git a/yarn.lock b/yarn.lock index dbec7aa3..1aa67acd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,7 +12,7 @@ __metadata: languageName: node linkType: hard -"@alwatr/async-queue@workspace:packages/async-queue": +"@alwatr/async-queue@workspace:^, @alwatr/async-queue@workspace:packages/async-queue": version: 0.0.0-use.local resolution: "@alwatr/async-queue@workspace:packages/async-queue" dependencies: @@ -84,7 +84,7 @@ __metadata: languageName: unknown linkType: soft -"@alwatr/flat-string@workspace:packages/flat-string": +"@alwatr/flat-string@workspace:^, @alwatr/flat-string@workspace:packages/flat-string": version: 0.0.0-use.local resolution: "@alwatr/flat-string@workspace:packages/flat-string" dependencies: @@ -119,7 +119,7 @@ __metadata: languageName: unknown linkType: soft -"@alwatr/logger@workspace:packages/logger": +"@alwatr/logger@workspace:^, @alwatr/logger@workspace:packages/logger": version: 0.0.0-use.local resolution: "@alwatr/logger@workspace:packages/logger" dependencies: @@ -146,6 +146,22 @@ __metadata: languageName: unknown linkType: soft +"@alwatr/node-fs@workspace:packages/node-fs": + version: 0.0.0-use.local + resolution: "@alwatr/node-fs@workspace:packages/node-fs" + dependencies: + "@alwatr/async-queue": "workspace:^" + "@alwatr/flat-string": "workspace:^" + "@alwatr/logger": "workspace:^" + "@alwatr/nano-build": "workspace:^" + "@alwatr/prettier-config": "workspace:^" + "@alwatr/tsconfig-base": "workspace:^" + "@alwatr/type-helper": "workspace:^" + "@types/node": "npm:^20.10.7" + typescript: "npm:^5.3.3" + languageName: unknown + linkType: soft + "@alwatr/platform-info@workspace:^, @alwatr/platform-info@workspace:packages/platform-info": version: 0.0.0-use.local resolution: "@alwatr/platform-info@workspace:packages/platform-info"