From f57846707130023b67896722c81b9a7e37d95bd6 Mon Sep 17 00:00:00 2001 From: pecodez Date: Thu, 15 Feb 2024 16:43:49 +0000 Subject: [PATCH] fix: adds large file support to bufferToSha1 method --- lib/analyzer/applications/java.ts | 32 +++++++++++++++---------------- lib/buffer-utils.ts | 21 +++++++++++++++----- test/lib/buffer-utils.spec.ts | 21 ++++++++++++++++++-- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/lib/analyzer/applications/java.ts b/lib/analyzer/applications/java.ts index 8fabcd26..aa10d400 100644 --- a/lib/analyzer/applications/java.ts +++ b/lib/analyzer/applications/java.ts @@ -43,7 +43,7 @@ export async function jarFilesToScannedResults( continue; } - const fingerprints = getFingerprints( + const fingerprints = await getFingerprints( desiredLevelsOfUnpacking, mappedResult[path], ); @@ -68,14 +68,15 @@ export async function jarFilesToScannedResults( return scanResults; } -function getFingerprints( +async function getFingerprints( desiredLevelsOfUnpacking: number, jarBuffers: JarBuffer[], -): JarFingerprint[] { - const fingerprints = new Set([ - ...unpackJars(jarBuffers, desiredLevelsOfUnpacking), - ]); - return Array.from(fingerprints); +): Promise { + const fingerprints: JarFingerprint[] = await unpackJars( + jarBuffers, + desiredLevelsOfUnpacking, + ); + return Array.from(new Set(fingerprints)); } /** @@ -188,11 +189,11 @@ function unpackJar({ * @param {number} unpackedLevels * @returns JarFingerprint[] */ -function unpackJars( +async function unpackJars( jarBuffers: JarBuffer[], desiredLevelsOfUnpacking: number, unpackedLevels: number = 0, -): JarFingerprint[] { +): Promise { // We have to unpack jars to get the pom.properties manifest which // we use to support shaded jars and get the package coords (if exists) // to reduce the dependency on maven search and support private jars. @@ -228,7 +229,7 @@ function unpackJars( // sha so maven-deps can fallback to searching maven central fingerprints.push({ location: jarInfo.location, - digest: jarInfo.coords ? null : bufferToSha1(jarInfo.buffer), + digest: jarInfo.coords ? null : await bufferToSha1(jarInfo.buffer), dependencies: jarInfo.dependencies, ...jarInfo.coords, }); @@ -237,13 +238,12 @@ function unpackJars( if (jarInfo.nestedJars.length > 0) { // this is an uber/fat JAR so we need to unpack the nested JARs to // analyze them for coords and further nested JARs (depth flag allowing) - fingerprints.push( - ...unpackJars( - jarInfo.nestedJars, - desiredLevelsOfUnpacking, - unpackedLevels + 1, - ), + const nestedJarFingerprints = await unpackJars( + jarInfo.nestedJars, + desiredLevelsOfUnpacking, + unpackedLevels + 1, ); + fingerprints.push(...nestedJarFingerprints); } } diff --git a/lib/buffer-utils.ts b/lib/buffer-utils.ts index 8c03b2cb..449312d6 100644 --- a/lib/buffer-utils.ts +++ b/lib/buffer-utils.ts @@ -1,12 +1,23 @@ import * as crypto from "crypto"; +import { Readable } from "stream"; import { HashAlgorithm } from "./types"; const HASH_ENCODING = "hex"; -export function bufferToSha1(buffer: Buffer): string { +export async function bufferToSha1(buffer: Buffer): Promise { + const stream = Readable.from(buffer); const hash = crypto.createHash(HashAlgorithm.Sha1); - hash.setEncoding(HASH_ENCODING); - hash.update(buffer); - hash.end(); - return hash.read().toString(HASH_ENCODING); + + return new Promise((resolve, reject) => { + stream + .pipe(hash) + .on("finish", () => { + hash.end(); + const digest = hash.read().toString(HASH_ENCODING); + resolve(digest); + }) + .on("error", (err) => { + reject(err); + }); + }); } diff --git a/test/lib/buffer-utils.spec.ts b/test/lib/buffer-utils.spec.ts index 6ab5c09a..e3740b0c 100644 --- a/test/lib/buffer-utils.spec.ts +++ b/test/lib/buffer-utils.spec.ts @@ -1,10 +1,11 @@ import { Buffer } from "buffer"; import * as crypto from "crypto"; + import { bufferToSha1 } from "../../lib/buffer-utils"; describe("buffer-utils", () => { describe("bufferToSha1", () => { - it("should convert Buffer to sha1", () => { + it("should convert Buffer to sha1", async () => { // Arrange const textToConvert = "hello world"; let hashedText = crypto.createHash("sha1"); @@ -16,10 +17,26 @@ describe("buffer-utils", () => { const bufferedText = Buffer.from(textToConvert); // Act - const result = bufferToSha1(bufferedText); + const result = await bufferToSha1(bufferedText); // Assert expect(result).toEqual(hashedText); }); + + it("should handle large files", async () => { + const megabyte = 1024 * 1024; + const gigabyte = megabyte * 1024; + + // create a buffer representing a file over 2GB, which would throw + // a RangeError if using the update method of a Crypto.Hash object, + // instead of the streaming interface which allows us to support + // large files + const buffer = Buffer.concat([ + Buffer.alloc(gigabyte * 2), + Buffer.alloc(megabyte), + ]); + const digest = await bufferToSha1(buffer); + expect(digest).toEqual(expect.any(String)); + }); }); });