From 84cae7be2d4c7234a706be9c30b2fa069ce1f402 Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Mon, 21 Feb 2022 17:05:52 +0000 Subject: [PATCH] feat: create read stream copy in case of file descriptor --- .../src/fsCreateReadStream.spec.ts | 46 +++++++++++++++++++ .../src/fsCreateReadStream.ts | 8 ++++ packages/hash-stream-node/src/isFileStream.ts | 3 +- .../src/readableStreamHasher.spec.ts | 9 ++-- .../src/readableStreamHasher.ts | 10 +--- 5 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 packages/hash-stream-node/src/fsCreateReadStream.spec.ts create mode 100644 packages/hash-stream-node/src/fsCreateReadStream.ts diff --git a/packages/hash-stream-node/src/fsCreateReadStream.spec.ts b/packages/hash-stream-node/src/fsCreateReadStream.spec.ts new file mode 100644 index 0000000000000..5a7e21a6b4b9c --- /dev/null +++ b/packages/hash-stream-node/src/fsCreateReadStream.spec.ts @@ -0,0 +1,46 @@ +import * as fs from "fs"; + +import { fsCreateReadStream } from "./fsCreateReadStream"; + +jest.setTimeout(30000); + +describe(fsCreateReadStream.name, () => { + const mockFileContents = fs.readFileSync(__filename, "utf8"); + + it("uses file descriptor if defined", (done) => { + fs.promises.open(__filename, "r").then((fd) => { + if ((fd as any).createReadStream) { + const readStream = (fd as any).createReadStream(); + const readStreamCopy = fsCreateReadStream(readStream); + + const chunks: Array = []; + readStreamCopy.on("data", (chunk) => { + chunks.push(chunk); + }); + readStreamCopy.on("end", () => { + const outputFileContents = Buffer.concat(chunks).toString(); + expect(outputFileContents).toEqual(mockFileContents); + done(); + }); + } else { + console.log(`Skipping createReadStream test as it's not available.`); + done(); + } + }); + }); + + it("uses start and end if file descriptor is not defined", (done) => { + const readStream = fs.createReadStream(__filename); + const readStreamCopy = fsCreateReadStream(readStream); + + const chunks: Array = []; + readStreamCopy.on("data", (chunk) => { + chunks.push(chunk); + }); + readStreamCopy.on("end", () => { + const outputFileContents = Buffer.concat(chunks).toString(); + expect(outputFileContents).toEqual(mockFileContents); + done(); + }); + }); +}); diff --git a/packages/hash-stream-node/src/fsCreateReadStream.ts b/packages/hash-stream-node/src/fsCreateReadStream.ts new file mode 100644 index 0000000000000..837f51b5cf1db --- /dev/null +++ b/packages/hash-stream-node/src/fsCreateReadStream.ts @@ -0,0 +1,8 @@ +import { createReadStream, ReadStream } from "fs"; + +export const fsCreateReadStream = (readStream: ReadStream) => + createReadStream(readStream.path, { + fd: (readStream as any).fd, + start: (readStream as any).start, + end: (readStream as any).end, + }); diff --git a/packages/hash-stream-node/src/isFileStream.ts b/packages/hash-stream-node/src/isFileStream.ts index e0016879c551f..3117afd0d9149 100644 --- a/packages/hash-stream-node/src/isFileStream.ts +++ b/packages/hash-stream-node/src/isFileStream.ts @@ -4,5 +4,4 @@ import { Readable } from "stream"; export const isFileStream = (stream: Readable): stream is ReadStream => typeof (stream as ReadStream).path === "string" || Buffer.isBuffer((stream as ReadStream).path) || - // @ts-expect-error fd is not defined in @types/node - typeof (stream as ReadStream).fd === "number"; + typeof (stream as any).fd === "number"; diff --git a/packages/hash-stream-node/src/readableStreamHasher.spec.ts b/packages/hash-stream-node/src/readableStreamHasher.spec.ts index 86c576b9b4c60..130e18051cf31 100644 --- a/packages/hash-stream-node/src/readableStreamHasher.spec.ts +++ b/packages/hash-stream-node/src/readableStreamHasher.spec.ts @@ -2,10 +2,12 @@ import { Hash } from "@aws-sdk/types"; import { createReadStream } from "fs"; import { Readable, Writable } from "stream"; +import { fsCreateReadStream } from "./fsCreateReadStream"; import { HashCalculator } from "./HashCalculator"; import { isFileStream } from "./isFileStream"; import { readableStreamHasher } from "./readableStreamHasher"; +jest.mock("./fsCreateReadStream"); jest.mock("./HashCalculator"); jest.mock("./isFileStream"); jest.mock("fs"); @@ -51,7 +53,7 @@ describe(readableStreamHasher.name, () => { }); it("creates a copy in case of fileStream", () => { - (createReadStream as jest.Mock).mockReturnValue( + (fsCreateReadStream as jest.Mock).mockReturnValue( new Readable({ read: (size) => {}, }) @@ -62,10 +64,7 @@ describe(readableStreamHasher.name, () => { readableStreamHasher(mockHashCtor, fsReadStream); expect(isFileStream).toHaveBeenCalledWith(fsReadStream); - expect(createReadStream).toHaveBeenCalledWith(fsReadStream.path, { - start: (fsReadStream as any).start, - end: (fsReadStream as any).end, - }); + expect(fsCreateReadStream).toHaveBeenCalledWith(fsReadStream); }); it("computes hash for a readable stream", async () => { diff --git a/packages/hash-stream-node/src/readableStreamHasher.ts b/packages/hash-stream-node/src/readableStreamHasher.ts index 29b41a07d401c..6ee1296091ddd 100644 --- a/packages/hash-stream-node/src/readableStreamHasher.ts +++ b/packages/hash-stream-node/src/readableStreamHasher.ts @@ -1,19 +1,13 @@ import { HashConstructor, StreamHasher } from "@aws-sdk/types"; -import { createReadStream } from "fs"; import { Readable } from "stream"; +import { fsCreateReadStream } from "./fsCreateReadStream"; import { HashCalculator } from "./HashCalculator"; import { isFileStream } from "./isFileStream"; export const readableStreamHasher: StreamHasher = (hashCtor: HashConstructor, readableStream: Readable) => { // ToDo: throw if readableStream is already flowing and it's copy can't be created. - // ToDo: create accurate copy if filestream is created from file descriptor. - const streamToPipe = isFileStream(readableStream) - ? createReadStream(readableStream.path, { - start: (readableStream as any).start, - end: (readableStream as any).end, - }) - : readableStream; + const streamToPipe = isFileStream(readableStream) ? fsCreateReadStream(readableStream) : readableStream; const hash = new hashCtor(); const hashCalculator = new HashCalculator(hash);