fix: upgrade data segment (#850)

Upgrades `data-segment` lib with Piece CID v2 storacha/data-segment#21 Needs: - [x] figure out legacy CID for storacha/data-segment#21 (review)
storacha · Aug 10, 2023 · fba281f · fba281f
1 parent b5138f7
commit fba281f
Show file tree

Hide file tree

Showing 16 changed files with 1,135 additions and 1,160 deletions.
diff --git a/packages/capabilities/package.json b/packages/capabilities/package.json
@@ -60,7 +60,7 @@
     "@ucanto/principal": "^8.0.0",
     "@ucanto/transport": "^8.0.0",
     "@ucanto/validator": "^8.0.0",
-    "@web3-storage/data-segment": "^2.2.0"
+    "@web3-storage/data-segment": "^3.0.1"
   },
   "devDependencies": {
     "@types/assert": "^1.5.6",

diff --git a/packages/capabilities/src/filecoin.js b/packages/capabilities/src/filecoin.js
@@ -13,13 +13,21 @@ import { capability, Schema, ok } from '@ucanto/validator'
 import { equal, equalWith, checkLink, and } from './utils.js'
 
 /**
- * @see https://github.com/multiformats/go-multihash/blob/dc3bd6897fcd17f6acd8d4d6ffd2cea3d4d3ebeb/multihash.go#L73
+ * @see https://github.com/filecoin-project/FIPs/pull/758/files
  */
-const SHA2_256_TRUNC254_PADDED = 0x1012
+const FR32_SHA2_256_TRUNC254_PADDED_BINARY_TREE = 0x1011
 /**
- * @see https://github.com/ipfs/go-cid/blob/829c826f6be23320846f4b7318aee4d17bf8e094/cid.go#L104
+ * @see https://github.com/filecoin-project/FIPs/pull/758/files
  */
-const FilCommitmentUnsealed = 0xf101
+const RAW_CODE = 0x55
+
+const PIECE_LINK = Schema.link({
+  code: RAW_CODE,
+  version: 1,
+  multihash: {
+    code: FR32_SHA2_256_TRUNC254_PADDED_BINARY_TREE,
+  },
+})
 
 /**
  * `filecoin/add` capability allows agent to add a filecoin piece to be aggregated
@@ -38,16 +46,10 @@ export const filecoinAdd = capability({
     content: Schema.link(),
     /**
      * CID of the piece.
+     *
+     * @see https://github.com/filecoin-project/FIPs/pull/758/files
      */
-    piece: /** @type {import('./types').PieceLinkSchema} */ (
-      Schema.link({
-        code: FilCommitmentUnsealed,
-        version: 1,
-        multihash: {
-          code: SHA2_256_TRUNC254_PADDED,
-        },
-      })
-    ),
+    piece: /** @type {import('./types').PieceLinkSchema} */ (PIECE_LINK),
   }),
   derives: (claim, from) => {
     return (
@@ -72,16 +74,10 @@ export const aggregateAdd = capability({
   nb: Schema.struct({
     /**
      * CID of the piece.
+     *
+     * @see https://github.com/filecoin-project/FIPs/pull/758/files
      */
-    piece: /** @type {import('./types').PieceLinkSchema} */ (
-      Schema.link({
-        code: FilCommitmentUnsealed,
-        version: 1,
-        multihash: {
-          code: SHA2_256_TRUNC254_PADDED,
-        },
-      })
-    ),
+    piece: /** @type {import('./types').PieceLinkSchema} */ (PIECE_LINK),
     /**
      * Storefront requestin piece to be aggregated
      */
@@ -120,17 +116,11 @@ export const dealAdd = capability({
     pieces: Schema.link(),
     /**
      * Commitment proof for the aggregate being offered.
-     * https://github.com/filecoin-project/go-state-types/blob/1e6cf0d47cdda75383ef036fc2725d1cf51dbde8/abi/piece.go#L47-L50
+     *
+     * @see https://github.com/filecoin-project/go-state-types/blob/1e6cf0d47cdda75383ef036fc2725d1cf51dbde8/abi/piece.go#L47-L50
+     * @see https://github.com/filecoin-project/FIPs/pull/758/files
      */
-    aggregate: /** @type {import('./types').PieceLinkSchema} */ (
-      Schema.link({
-        code: FilCommitmentUnsealed,
-        version: 1,
-        multihash: {
-          code: SHA2_256_TRUNC254_PADDED,
-        },
-      })
-    ),
+    aggregate: /** @type {import('./types').PieceLinkSchema} */ (PIECE_LINK),
     /**
      * Storefront requesting deal
      */
@@ -164,16 +154,10 @@ export const chainTrackerInfo = capability({
   nb: Schema.struct({
     /**
      * CID of the piece.
+     *
+     * @see https://github.com/filecoin-project/FIPs/pull/758/files
      */
-    piece: /** @type {import('./types').PieceLinkSchema} */ (
-      Schema.link({
-        code: FilCommitmentUnsealed,
-        version: 1,
-        multihash: {
-          code: SHA2_256_TRUNC254_PADDED,
-        },
-      })
-    ),
+    piece: /** @type {import('./types').PieceLinkSchema} */ (PIECE_LINK),
   }),
   derives: (claim, from) => {
     return (

diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts
@@ -33,6 +33,9 @@ export interface UnknownProvider extends Ucanto.Failure {
   did: DID
 }
 
+/**
+ * @see https://github.com/filecoin-project/FIPs/pull/758/files
+ */
 export type PieceLinkSchema = Schema.Schema<PieceLink>
 
 // Access

diff --git a/packages/filecoin-api/package.json b/packages/filecoin-api/package.json
@@ -86,7 +86,7 @@
     "@ucanto/server": "^8.0.0",
     "@ucanto/transport": "^8.0.0",
     "@web3-storage/capabilities": "workspace:^",
-    "@web3-storage/data-segment": "^2.2.0"
+    "@web3-storage/data-segment": "^3.0.1"
   },
   "devDependencies": {
     "@ipld/car": "^5.1.1",

diff --git a/packages/filecoin-api/test/services/aggregator.js b/packages/filecoin-api/test/services/aggregator.js
@@ -39,7 +39,7 @@ export const test = {
       throw new Error('invocation failed', { cause: response.out.error })
     }
     assert.ok(response.out.ok)
-    assert.deepEqual(response.out.ok.piece, cargo.link.link())
+    assert.ok(response.out.ok.piece.equals(cargo.link.link()))
 
     // Validate effect in receipt
     const fx = await Filecoin.aggregateAdd
@@ -98,7 +98,7 @@ export const test = {
       throw new Error('invocation failed', { cause: response.out.error })
     }
     assert.ok(response.out.ok)
-    assert.deepEqual(response.out.ok.piece, cargo.link.link())
+    assert.ok(response.out.ok.piece.equals(cargo.link.link()))
 
     // Validate queue and store
     await pWaitFor(() => context.queuedMessages.length === 0)
@@ -141,7 +141,7 @@ export const test = {
         throw new Error('invocation failed', { cause: response.out.error })
       }
       assert.ok(response.out.ok)
-      assert.deepEqual(response.out.ok.piece, cargo.link.link())
+      assert.ok(response.out.ok.piece.equals(cargo.link.link()))
 
       // Validate queue and store
       await pWaitFor(() => context.queuedMessages.length === 0)

diff --git a/packages/filecoin-api/test/services/dealer.js b/packages/filecoin-api/test/services/dealer.js
@@ -49,7 +49,7 @@ export const test = {
       throw new Error('invocation failed', { cause: response.out.error })
     }
     assert.ok(response.out.ok)
-    assert.deepEqual(response.out.ok.aggregate, aggregate.link)
+    assert.ok(response.out.ok.aggregate?.equals(aggregate.link))
 
     // Validate effect in receipt
     const fx = await Filecoin.dealAdd
@@ -111,7 +111,7 @@ export const test = {
         throw new Error('invocation failed', { cause: response.out.error })
       }
       assert.ok(response.out.ok)
-      assert.deepEqual(response.out.ok.aggregate, aggregate.link)
+      assert.ok(response.out.ok.aggregate?.equals(aggregate.link))
 
       // Validate queue and store
       await pWaitFor(() => context.queuedMessages.length === 0)

diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js
@@ -40,7 +40,7 @@ export const test = {
       throw new Error('invocation failed', { cause: response.out.error })
     }
     assert.ok(response.out.ok)
-    assert.deepEqual(response.out.ok.piece, cargo.link.link())
+    assert.ok(response.out.ok.piece.equals(cargo.link.link()))
 
     // Validate effect in receipt
     const fx = await Filecoin.filecoinAdd
@@ -92,7 +92,7 @@ export const test = {
         throw new Error('invocation failed', { cause: response.out.error })
       }
       assert.ok(response.out.ok)
-      assert.deepEqual(response.out.ok.piece, cargo.link.link())
+      assert.ok(response.out.ok.piece.equals(cargo.link.link()))
 
       // Validate queue and store
       await pWaitFor(() => context.queuedMessages.length === 0)
@@ -128,7 +128,7 @@ export const test = {
         throw new Error('invocation failed', { cause: response.out.error })
       }
       assert.ok(response.out.ok)
-      assert.deepEqual(response.out.ok.piece, cargo.link.link())
+      assert.ok(response.out.ok.piece.equals(cargo.link.link()))
 
       // Validate queue and store
       await pWaitFor(() => context.queuedMessages.length === 0)

diff --git a/packages/filecoin-api/test/utils.js b/packages/filecoin-api/test/utils.js
@@ -63,13 +63,13 @@ export async function randomCargo(length, size) {
   )
 
   return cars.map((car) => {
-    const piece = Piece.build(car.bytes)
+    const piece = Piece.fromPayload(car.bytes)
 
     return {
       link: piece.link,
-      content: car.cid,
       height: piece.height,
-      size: piece.size,
+      root: piece.root,
+      content: car.cid,
     }
   })
 }
@@ -80,19 +80,12 @@ export async function randomCargo(length, size) {
  */
 export async function randomAggregate(length, size) {
   const pieces = await randomCargo(length, size)
-
   const aggregateBuild = Aggregate.build({
     pieces,
   })
 
   return {
-    pieces: pieces.map((p) => ({
-      link: p.link,
-      height: p.height,
-    })),
-    aggregate: {
-      link: aggregateBuild.link,
-      height: aggregateBuild.height,
-    },
+    pieces,
+    aggregate: aggregateBuild,
   }
 }
diff --git a/packages/filecoin-client/package.json b/packages/filecoin-client/package.json
@@ -69,7 +69,7 @@
     "@types/mocha": "^10.0.1",
     "@ucanto/principal": "^8.0.0",
     "@ucanto/server": "^8.0.1",
-    "@web3-storage/data-segment": "^2.2.0",
+    "@web3-storage/data-segment": "^3.0.1",
     "assert": "^2.0.0",
     "c8": "^7.13.0",
     "hd-scripts": "^4.0.0",

diff --git a/packages/filecoin-client/test/aggregator.test.js b/packages/filecoin-client/test/aggregator.test.js
@@ -75,7 +75,7 @@ describe('aggregate/add', () => {
     )
 
     assert.ok(res.out.ok)
-    assert.deepEqual(res.out.ok, pieceAddResponse)
+    assert.ok(res.out.ok.piece.equals(pieceAddResponse.piece))
     // includes effect fx in receipt
     assert.ok(res.fx.join)
   })
@@ -132,7 +132,7 @@ describe('aggregate/add', () => {
     )
 
     assert.ok(res.out.ok)
-    assert.deepEqual(res.out.ok, pieceAddResponse)
+    assert.ok(res.out.ok.piece.equals(pieceAddResponse.piece))
     // does not include effect fx in receipt
     assert.ok(!res.fx.join)
   })

diff --git a/packages/filecoin-client/test/chain-tracker.test.js b/packages/filecoin-client/test/chain-tracker.test.js
@@ -59,7 +59,8 @@ describe('chain.info', () => {
     )
 
     assert.ok(res.out.ok)
-    assert.deepEqual(res.out.ok, chainInfoResponse)
+    // @ts-expect-error todo check error
+    assert.ok(res.out.ok.piece.equals(chainInfoResponse.piece))
   })
 })
 

diff --git a/packages/filecoin-client/test/dealer.test.js b/packages/filecoin-client/test/dealer.test.js
@@ -11,10 +11,10 @@ import { dealAdd } from '../src/dealer.js'
 import { randomAggregate } from './helpers/random.js'
 import { mockService } from './helpers/mocks.js'
 import { OperationFailed, OperationErrorName } from './helpers/errors.js'
-import { serviceProvider as brokerService } from './fixtures.js'
+import { serviceProvider as dealerService } from './fixtures.js'
 
-describe('aggregate.add', () => {
-  it('aggregator adds an aggregate piece to the broker, getting the piece queued', async () => {
+describe('dealer.add', () => {
+  it('aggregator adds an aggregate piece to the dealer, getting the piece queued', async () => {
     const { aggregator, storefront: storefrontSigner } = await getContext()
 
     // generate aggregate to add
@@ -71,7 +71,7 @@ describe('aggregate.add', () => {
       {
         issuer: aggregator,
         with: aggregator.did(),
-        audience: brokerService,
+        audience: dealerService,
       },
       aggregate.link.link(),
       offer,
@@ -81,12 +81,12 @@ describe('aggregate.add', () => {
     )
 
     assert.ok(res.out.ok)
-    assert.deepEqual(res.out.ok, dealAddResponse)
+    assert.ok(res.out.ok.aggregate?.equals(dealAddResponse.aggregate))
     // includes effect fx in receipt
     assert.ok(res.fx.join)
   })
 
-  it('broker self invokes add an aggregate piece to accept the piece queued', async () => {
+  it('dealer self invokes add an aggregate piece to accept the piece queued', async () => {
     const { storefront: storefrontSigner } = await getContext()
 
     // generate aggregate to add
@@ -107,7 +107,7 @@ describe('aggregate.add', () => {
         add: Server.provideAdvanced({
           capability: FilecoinCapabilities.dealAdd,
           handler: async ({ invocation }) => {
-            assert.strictEqual(invocation.issuer.did(), brokerService.did())
+            assert.strictEqual(invocation.issuer.did(), dealerService.did())
             assert.strictEqual(invocation.capabilities.length, 1)
             const invCap = invocation.capabilities[0]
             assert.strictEqual(invCap.can, FilecoinCapabilities.dealAdd.can)
@@ -132,9 +132,9 @@ describe('aggregate.add', () => {
     // invoke piece add from storefront
     const res = await dealAdd(
       {
-        issuer: brokerService,
-        with: brokerService.did(),
-        audience: brokerService,
+        issuer: dealerService,
+        with: dealerService.did(),
+        audience: dealerService,
       },
       aggregate.link.link(),
       offer,
@@ -144,12 +144,12 @@ describe('aggregate.add', () => {
     )
 
     assert.ok(res.out.ok)
-    assert.deepEqual(res.out.ok, dealAddResponse)
+    assert.ok(res.out.ok.aggregate?.equals(dealAddResponse.aggregate))
     // does not include effect fx in receipt
     assert.ok(!res.fx.join)
   })
 
-  it('broker self invokes add an aggregate piece to reject the piece queued', async () => {
+  it('dealer self invokes add an aggregate piece to reject the piece queued', async () => {
     const { storefront: storefrontSigner } = await getContext()
 
     // generate aggregate to add
@@ -171,7 +171,7 @@ describe('aggregate.add', () => {
         add: Server.provideAdvanced({
           capability: FilecoinCapabilities.dealAdd,
           handler: async ({ invocation, context }) => {
-            assert.strictEqual(invocation.issuer.did(), brokerService.did())
+            assert.strictEqual(invocation.issuer.did(), dealerService.did())
             assert.strictEqual(invocation.capabilities.length, 1)
             const invCap = invocation.capabilities[0]
             assert.strictEqual(invCap.can, FilecoinCapabilities.dealAdd.can)
@@ -198,9 +198,9 @@ describe('aggregate.add', () => {
     // invoke piece add from storefront
     const res = await dealAdd(
       {
-        issuer: brokerService,
-        with: brokerService.did(),
-        audience: brokerService,
+        issuer: dealerService,
+        with: dealerService.did(),
+        audience: dealerService,
       },
       aggregate.link.link(),
       offer,
@@ -228,12 +228,12 @@ async function getContext() {
  */
 function getConnection(service) {
   const server = Server.create({
-    id: brokerService,
+    id: dealerService,
     service,
     codec: CAR.inbound,
   })
   const connection = Client.connect({
-    id: brokerService,
+    id: dealerService,
     codec: CAR.outbound,
     channel: server,
   })