From 1275027c42c27b901ba9fb1cd9977da1fc9fedfa Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Mon, 4 Mar 2024 16:46:33 +0000
Subject: [PATCH] feat: byo piece hasher

---
 packages/upload-client/package.json |  2 +-
 packages/upload-client/src/index.js |  5 +++--
 packages/upload-client/src/types.ts |  4 +++-
 pnpm-lock.yaml                      | 18 +++++++++++-------
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json
index 5a86b5f21..b7ffc8a8a 100644
--- a/packages/upload-client/package.json
+++ b/packages/upload-client/package.json
@@ -75,7 +75,7 @@
     "@ucanto/interface": "^9.0.0",
     "@ucanto/transport": "^9.1.0",
     "@web3-storage/capabilities": "workspace:^",
-    "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0",
+    "@web3-storage/data-segment": "^5.1.0",
     "ipfs-utils": "^9.0.14",
     "multiformats": "^12.1.2",
     "p-retry": "^5.1.2",
diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js
index 6c6233d64..624668cfa 100644
--- a/packages/upload-client/src/index.js
+++ b/packages/upload-client/src/index.js
@@ -1,5 +1,5 @@
 import { Parallel } from 'parallel-transform-web'
-import * as PieceHasher from 'fr32-sha2-256-trunc254-padded-binary-tree-multihash/async'
+import * as PieceHasher from '@web3-storage/data-segment/multihash'
 import * as Link from 'multiformats/link'
 import * as raw from 'multiformats/codecs/raw'
 import * as Store from './store.js'
@@ -123,6 +123,7 @@ async function uploadBlockStream(conf, blocks, options = {}) {
   /** @type {import('./types.js').AnyLink?} */
   let root = null
   const concurrency = options.concurrentRequests ?? CONCURRENT_REQUESTS
+  const hasher = options.pieceHasher ?? PieceHasher
   await blocks
     .pipeThrough(new ShardingStream(options))
     .pipeThrough(
@@ -131,7 +132,7 @@ async function uploadBlockStream(conf, blocks, options = {}) {
         const [cid, piece] = await Promise.all([
           Store.add(conf, bytes, options),
           (async () => {
-            const multihashDigest = await PieceHasher.digest(bytes)
+            const multihashDigest = await hasher.digest(bytes)
             return /** @type {import('@web3-storage/capabilities/types').PieceLink} */ (
               Link.create(raw.code, multihashDigest)
             )
diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts
index cc066729e..9954e0c34 100644
--- a/packages/upload-client/src/types.ts
+++ b/packages/upload-client/src/types.ts
@@ -2,7 +2,7 @@ import type {
   FetchOptions as IpfsUtilsFetchOptions,
   ProgressStatus as XHRProgressStatus,
 } from 'ipfs-utils/src/types.js'
-import { Link, UnknownLink, Version } from 'multiformats/link'
+import { Link, UnknownLink, Version, MultihashHasher } from 'multiformats'
 import { Block } from '@ipld/unixfs'
 import {
   ServiceMethod,
@@ -44,6 +44,7 @@ import {
   UsageReportSuccess,
   UsageReportFailure,
 } from '@web3-storage/capabilities/types'
+import { code as pieceHashCode } from '@web3-storage/data-segment/multihash'
 
 type Override<T, R> = Omit<T, keyof R> & R
 
@@ -271,6 +272,7 @@ export interface UploadOptions
     ShardStoringOptions,
     UploadProgressTrackable {
   onShardStored?: (meta: CARMetadata) => void
+  pieceHasher?: MultihashHasher<typeof pieceHashCode>
 }
 
 export interface UploadDirectoryOptions
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c2ac72c1f..f7d4a9386 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -467,9 +467,9 @@ importers:
       '@web3-storage/capabilities':
         specifier: workspace:^
         version: link:../capabilities
-      fr32-sha2-256-trunc254-padded-binary-tree-multihash:
-        specifier: ^3.3.0
-        version: 3.3.0
+      '@web3-storage/data-segment':
+        specifier: ^5.1.0
+        version: 5.1.0
       ipfs-utils:
         specifier: ^9.0.14
         version: 9.0.14
@@ -4411,6 +4411,14 @@ packages:
       sync-multihash-sha2: 1.0.0
     dev: true
 
+  /@web3-storage/data-segment@5.1.0:
+    resolution: {integrity: sha512-FYdmtKvNiVz+maZ++k4PdD43rfJW5DeagLpstq2y84CyOKNRBWbHLCZ/Ec5zT9iGI+0WgsCGbpC/WlG0jlrnhA==}
+    dependencies:
+      '@ipld/dag-cbor': 9.0.6
+      multiformats: 11.0.2
+      sync-multihash-sha2: 1.0.0
+    dev: false
+
   /@web3-storage/sigv4@1.0.2:
     resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==}
     dependencies:
@@ -7063,10 +7071,6 @@ packages:
     engines: {node: '>= 0.6'}
     dev: true
 
-  /fr32-sha2-256-trunc254-padded-binary-tree-multihash@3.3.0:
-    resolution: {integrity: sha512-O11VDxPmPvbQj5eac2BJXyieNacyd+RCMhwOzXQQM/NCI25x3c32YWB4/JwgOWPCpKnNXF6lpK/j0lj7GWOnYQ==}
-    dev: false
-
   /fraction.js@4.3.7:
     resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==}
     dev: true