fix: adds large file support to bufferToSha1 method

lib/analyzer/applications/java.ts

@@ -43,7 +43,7 @@ export async function jarFilesToScannedResults(
       continue;
     }
 
-    const fingerprints = getFingerprints(
+    const fingerprints = await getFingerprints(
       desiredLevelsOfUnpacking,
       mappedResult[path],
     );
@@ -68,14 +68,15 @@ export async function jarFilesToScannedResults(
   return scanResults;
 }
 
-function getFingerprints(
+async function getFingerprints(
   desiredLevelsOfUnpacking: number,
   jarBuffers: JarBuffer[],
-): JarFingerprint[] {
-  const fingerprints = new Set([
-    ...unpackJars(jarBuffers, desiredLevelsOfUnpacking),
-  ]);
-  return Array.from(fingerprints);
+): Promise<JarFingerprint[]> {
+  const fingerprints: JarFingerprint[] = await unpackJars(
+    jarBuffers,
+    desiredLevelsOfUnpacking,
+  );
+  return Array.from(new Set(fingerprints));
 }
 
 /**
@@ -188,11 +189,11 @@ function unpackJar({
  * @param {number} unpackedLevels
  * @returns JarFingerprint[]
  */
-function unpackJars(
+async function unpackJars(
   jarBuffers: JarBuffer[],
   desiredLevelsOfUnpacking: number,
   unpackedLevels: number = 0,
-): JarFingerprint[] {
+): Promise<JarFingerprint[]> {
   // We have to unpack jars to get the pom.properties manifest which
   // we use to support shaded jars and get the package coords (if exists)
   // to reduce the dependency on maven search and support private jars.
@@ -228,7 +229,7 @@ function unpackJars(
       // sha so maven-deps can fallback to searching maven central
       fingerprints.push({
         location: jarInfo.location,
-        digest: jarInfo.coords ? null : bufferToSha1(jarInfo.buffer),
+        digest: jarInfo.coords ? null : await bufferToSha1(jarInfo.buffer),
         dependencies: jarInfo.dependencies,
         ...jarInfo.coords,
       });
@@ -237,13 +238,12 @@ function unpackJars(
     if (jarInfo.nestedJars.length > 0) {
       // this is an uber/fat JAR so we need to unpack the nested JARs to
       // analyze them for coords and further nested JARs (depth flag allowing)
-      fingerprints.push(
-        ...unpackJars(
-          jarInfo.nestedJars,
-          desiredLevelsOfUnpacking,
-          unpackedLevels + 1,
-        ),
+      const nestedJarFingerprints = await unpackJars(
+        jarInfo.nestedJars,
+        desiredLevelsOfUnpacking,
+        unpackedLevels + 1,
       );
+      fingerprints.push(...nestedJarFingerprints);
     }
   }
 

lib/buffer-utils.ts

@@ -1,12 +1,23 @@
 import * as crypto from "crypto";
+import { Readable } from "stream";
 import { HashAlgorithm } from "./types";
 
 const HASH_ENCODING = "hex";
 
-export function bufferToSha1(buffer: Buffer): string {
+export async function bufferToSha1(buffer: Buffer): Promise<string> {
+  const stream = Readable.from(buffer);
   const hash = crypto.createHash(HashAlgorithm.Sha1);
-  hash.setEncoding(HASH_ENCODING);
-  hash.update(buffer);
-  hash.end();
-  return hash.read().toString(HASH_ENCODING);
+
+  return new Promise((resolve, reject) => {
+    stream
+      .pipe(hash)
+      .on("finish", () => {
+        hash.end();
+        const digest = hash.read().toString(HASH_ENCODING);
+        resolve(digest);
+      })
+      .on("error", (err) => {
+        reject(err);
+      });
+  });
 }

test/lib/buffer-utils.spec.ts

@@ -1,10 +1,11 @@
 import { Buffer } from "buffer";
 import * as crypto from "crypto";
+
 import { bufferToSha1 } from "../../lib/buffer-utils";
 
 describe("buffer-utils", () => {
   describe("bufferToSha1", () => {
-    it("should convert Buffer to sha1", () => {
+    it("should convert Buffer to sha1", async () => {
       // Arrange
       const textToConvert = "hello world";
       let hashedText = crypto.createHash("sha1");
@@ -16,10 +17,26 @@ describe("buffer-utils", () => {
       const bufferedText = Buffer.from(textToConvert);
 
       // Act
-      const result = bufferToSha1(bufferedText);
+      const result = await bufferToSha1(bufferedText);
 
       // Assert
       expect(result).toEqual(hashedText);
     });
+
+    it("should handle large files", async () => {
+      const megabyte = 1024 * 1024;
+      const gigabyte = megabyte * 1024;
+
+      // create a buffer representing a file over 2GB, which would throw
+      // a RangeError if using the update method of a Crypto.Hash object,
+      // instead of the streaming interface which allows us to support
+      // large files
+      const buffer = Buffer.concat([
+        Buffer.alloc(gigabyte * 2),
+        Buffer.alloc(megabyte),
+      ]);
+      const digest = await bufferToSha1(buffer);
+      expect(digest).toEqual(expect.any(String));
+    });
   });
 });