feat!: add size to store/remove receipt (#969)

This PR adds a `size` property to the `store/remove` receipt. This is so we can account for removals properly. This also moves the `store/*` and `upload/*` receipt types into the `@web3-storage/capabilities` package to be consistent with the rest of them. This accounts for much of the churn here. TODO: update https://github.com/web3-storage/w3infra/blob/9f0c50dd2cf4adec616483d7c8d29cfdec03129c/ucan-invocation/functions/space-metrics-store-remove-size-total.js#L43-L51 (in separate PR) BREAKING CHANGE: Returning the `size` means that we need to fetch the stored item beforehand, and if it does not exist throw a `StoreItemNotFound` error. This is a change from the current behaviour which returns successfully even if the item is not present in the space.
storacha · Oct 12, 2023 · d2100eb · d2100eb
1 parent e478333
commit d2100eb
Show file tree

Hide file tree

Showing 22 changed files with 223 additions and 190 deletions.
diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts
@@ -1,7 +1,16 @@
 import type { TupleToUnion } from 'type-fest'
 import * as Ucanto from '@ucanto/interface'
 import type { Schema } from '@ucanto/core'
-import { InferInvokedCapability, Unit, DID, DIDKey } from '@ucanto/interface'
+import {
+  InferInvokedCapability,
+  Unit,
+  DID,
+  DIDKey,
+  ToString,
+  Link,
+  UnknownLink,
+} from '@ucanto/interface'
+import { CAR } from '@ucanto/transport'
 import type { PieceLink } from '@web3-storage/data-segment'
 import { space, info } from './space.js'
 import * as provider from './provider.js'
@@ -19,6 +28,11 @@ import * as UCANCaps from './ucan.js'
 
 export type { Unit, PieceLink }
 
+/**
+ * An IPLD Link that has the CAR codec code.
+ */
+export type CARLink = Link<unknown, typeof CAR.codec.code>
+
 export type AccountDID = DID<'mailto'>
 export type SpaceDID = DID<'key'>
 
@@ -209,11 +223,74 @@ export type Upload = InferInvokedCapability<typeof UploadCaps.upload>
 export type UploadAdd = InferInvokedCapability<typeof UploadCaps.add>
 export type UploadRemove = InferInvokedCapability<typeof UploadCaps.remove>
 export type UploadList = InferInvokedCapability<typeof UploadCaps.list>
+
 // Store
 export type Store = InferInvokedCapability<typeof store>
 export type StoreAdd = InferInvokedCapability<typeof add>
 export type StoreRemove = InferInvokedCapability<typeof remove>
 export type StoreList = InferInvokedCapability<typeof list>
+
+export type StoreAddSuccess = StoreAddSuccessDone | StoreAddSuccessUpload
+export interface StoreAddSuccessDone {
+  status: 'done'
+  with: DID
+  link: UnknownLink
+  url?: undefined
+  headers?: undefined
+}
+
+export interface StoreAddSuccessUpload {
+  status: 'upload'
+  with: DID
+  link: UnknownLink
+  url: ToString<URL>
+  headers: Record<string, string>
+}
+
+export interface StoreRemoveSuccess {
+  size: number
+}
+
+export interface StoreItemNotFound extends Ucanto.Failure {
+  name: 'StoreItemNotFound'
+}
+
+export type StoreRemoveFailure = StoreItemNotFound | Ucanto.Failure
+
+export interface StoreListSuccess extends ListResponse<StoreListItem> {}
+
+export interface ListResponse<R> {
+  cursor?: string
+  before?: string
+  after?: string
+  size: number
+  results: R[]
+}
+
+export interface StoreListItem {
+  link: UnknownLink
+  size: number
+  origin?: UnknownLink
+}
+
+export interface UploadAddSuccess {
+  root: UnknownLink
+  shards?: CARLink[]
+}
+
+export type UploadRemoveSuccess = UploadDidRemove | UploadDidNotRemove
+
+export interface UploadDidRemove extends UploadAddSuccess {}
+
+export interface UploadDidNotRemove {
+  root?: undefined
+  shards?: undefined
+}
+
+export interface UploadListSuccess extends ListResponse<UploadListItem> {}
+
+export interface UploadListItem extends UploadAddSuccess {}
+
 // UCAN core events
 
 export type UCANRevoke = InferInvokedCapability<typeof UCANCaps.revoke>

diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js
@@ -5,7 +5,7 @@ import { allocate } from '../space-allocate.js'
 
 /**
  * @param {API.StoreServiceContext} context
- * @returns {API.ServiceMethod<API.StoreAdd, API.StoreAddOk, API.Failure>}
+ * @returns {API.ServiceMethod<API.StoreAdd, API.StoreAddSuccess, API.Failure>}
  */
 export function storeAddProvider(context) {
   const { storeTable, carStoreBucket, maxUploadSize } = context
@@ -69,7 +69,7 @@ export function storeAddProvider(context) {
         status: 'upload',
         with: space,
         link,
-        url,
+        url: url.toString(),
         headers,
       },
     }

diff --git a/packages/upload-api/src/store/list.js b/packages/upload-api/src/store/list.js
@@ -4,7 +4,7 @@ import * as API from '../types.js'
 
 /**
  * @param {API.StoreServiceContext} context
- * @returns {API.ServiceMethod<API.StoreList, API.StoreListOk, API.Failure>}
+ * @returns {API.ServiceMethod<API.StoreList, API.StoreListSuccess, API.Failure>}
  */
 export function storeListProvider(context) {
   return Server.provide(Store.list, async ({ capability }) => {

diff --git a/packages/upload-api/src/store/remove.js b/packages/upload-api/src/store/remove.js
@@ -4,17 +4,48 @@ import * as API from '../types.js'
 
 /**
  * @param {API.StoreServiceContext} context
- * @returns {API.ServiceMethod<API.StoreRemove, API.Unit, API.Failure>}
+ * @returns {API.ServiceMethod<API.StoreRemove, API.StoreRemoveSuccess, API.StoreRemoveFailure>}
  */
 export function storeRemoveProvider(context) {
   return Server.provide(Store.remove, async ({ capability }) => {
     const { link } = capability.nb
     const space = Server.DID.parse(capability.with).did()
 
+    const item = await context.storeTable.get(space, link)
+    if (!item) {
+      return Server.error(new StoreItemNotFound(space, link))
+    }
+
     await context.storeTable.remove(space, link)
 
+    return Server.ok({ size: item.size })
+  })
+}
+
+class StoreItemNotFound extends Server.Failure {
+  /**
+   * @param {import('@ucanto/interface').DID} space
+   * @param {import('@ucanto/interface').UnknownLink} link
+   */
+  constructor(space, link) {
+    super()
+    this.space = space
+    this.link = link
+  }
+
+  get name() {
+    return 'StoreItemNotFound'
+  }
+
+  describe() {
+    return `${this.link} not found in ${this.space}`
+  }
+
+  toJSON() {
     return {
-      ok: {},
+      ...super.toJSON(),
+      space: this.space,
+      link: { '/': this.link.toString() },
     }
-  })
+  }
 }
diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts
@@ -8,17 +8,17 @@ import type {
   DIDKey,
   InboundCodec,
   Result,
-  Unit,
   CapabilityParser,
   Match,
   ParsedCapability,
   InferInvokedCapability,
   RevocationChecker,
+  ToString,
+  UnknownLink,
 } from '@ucanto/interface'
 import type { ProviderInput, ConnectionView } from '@ucanto/server'
 
 import { Signer as EdSigner } from '@ucanto/principal/ed25519'
-import { ToString, UnknownLink } from 'multiformats'
 import { DelegationsStorage as Delegations } from './types/delegations'
 import { ProvisionsStorage as Provisions } from './types/provisions'
 import { RateLimitsStorage as RateLimits } from './types/rate-limits'
@@ -59,11 +59,20 @@ export interface DebugEmail extends Email {
 
 import {
   StoreAdd,
+  StoreAddSuccess,
   StoreRemove,
+  StoreRemoveSuccess,
+  StoreRemoveFailure,
   StoreList,
+  StoreListSuccess,
+  StoreListItem,
   UploadAdd,
+  UploadAddSuccess,
   UploadRemove,
+  UploadRemoveSuccess,
   UploadList,
+  UploadListSuccess,
+  UploadListItem,
   AccessAuthorize,
   AccessAuthorizeSuccess,
   AccessDelegate,
@@ -108,6 +117,8 @@ import {
   SpaceInfo,
   ProviderDID,
   UCANRevoke,
+  ListResponse,
+  CARLink,
 } from '@web3-storage/capabilities/types'
 import * as Capabilities from '@web3-storage/capabilities'
 import { RevocationsStorage } from './types/revocations'
@@ -130,14 +141,14 @@ export type { RateLimitsStorage, RateLimit } from './types/rate-limits'
 
 export interface Service {
   store: {
-    add: ServiceMethod<StoreAdd, StoreAddOk, Failure>
-    remove: ServiceMethod<StoreRemove, Unit, Failure>
-    list: ServiceMethod<StoreList, StoreListOk, Failure>
+    add: ServiceMethod<StoreAdd, StoreAddSuccess, Failure>
+    remove: ServiceMethod<StoreRemove, StoreRemoveSuccess, StoreRemoveFailure>
+    list: ServiceMethod<StoreList, StoreListSuccess, Failure>
   }
   upload: {
-    add: ServiceMethod<UploadAdd, UploadAddOk, Failure>
-    remove: ServiceMethod<UploadRemove, UploadRemoveOk, Failure>
-    list: ServiceMethod<UploadList, UploadListOk, Failure>
+    add: ServiceMethod<UploadAdd, UploadAddSuccess, Failure>
+    remove: ServiceMethod<UploadRemove, UploadRemoveSuccess, Failure>
+    list: ServiceMethod<UploadList, UploadListSuccess, Failure>
   }
   console: {
     log: ServiceMethod<
@@ -363,6 +374,7 @@ export interface DudewhereBucket {
 export interface StoreTable {
   inspect: (link: UnknownLink) => Promise<StoreGetOk>
   exists: (space: DID, link: UnknownLink) => Promise<boolean>
+  get: (space: DID, link: UnknownLink) => Promise<StoreAddOutput | undefined>
   insert: (item: StoreAddInput) => Promise<StoreAddOutput>
   remove: (space: DID, link: UnknownLink) => Promise<void>
   list: (
@@ -375,18 +387,22 @@ export interface TestStoreTable {
   get(
     space: DID,
     link: UnknownLink
-  ): Promise<(StoreAddInput & StoreListItem) | undefined>
+  ): Promise<
+    (StoreAddInput & StoreListItem & { insertedAt: string }) | undefined
+  >
 }
 
 export interface UploadTable {
   inspect: (link: UnknownLink) => Promise<UploadGetOk>
   exists: (space: DID, root: UnknownLink) => Promise<boolean>
-  insert: (item: UploadAddInput) => Promise<UploadAddOk>
-  remove: (space: DID, root: UnknownLink) => Promise<UploadRemoveOk | null>
+  insert: (item: UploadAddInput) => Promise<UploadAddSuccess>
+  remove: (space: DID, root: UnknownLink) => Promise<UploadRemoveSuccess | null>
   list: (
     space: DID,
     options?: ListOptions
-  ) => Promise<ListResponse<UploadListItem>>
+  ) => Promise<
+    ListResponse<UploadListItem & { insertedAt: string; updatedAt: string }>
+  >
 }
 
 export type SpaceInfoSuccess = {
@@ -428,74 +444,24 @@ export interface StoreGetOk {
   spaces: Array<{ did: DID; insertedAt: string }>
 }
 
-export interface StoreListItem extends StoreAddOutput {
-  insertedAt: string
-}
-
-export interface StoreListOk extends ListResponse<StoreListItem> {}
-
-export type StoreAddOk = StoreAddDone | StoreAddUpload
-
-export interface StoreAddDone {
-  status: 'done'
-  with: DID
-  link: UnknownLink
-  url?: undefined
-  headers?: undefined
-}
-
-export interface StoreAddUpload {
-  status: 'upload'
-  with: DID
-  link: UnknownLink
-  url: URL
-  headers: Record<string, string>
-}
-
 export interface UploadAddInput {
   space: DID
   root: UnknownLink
-  shards?: UnknownLink[]
+  shards?: CARLink[]
   issuer: DID
   invocation: UCANLink
 }
 
-export interface UploadAddOk
-  extends Omit<UploadAddInput, 'space' | 'issuer' | 'invocation'> {}
-export type UploadRemoveOk = UploadDIDRemove | UploadDidNotRemove
-
-export interface UploadDIDRemove extends UploadAddOk {}
-export interface UploadDidNotRemove {
-  root?: undefined
-  shards?: undefined
-}
-
 export interface UploadGetOk {
   spaces: Array<{ did: DID; insertedAt: string }>
 }
 
-export interface UploadListItem extends UploadAddOk {
-  insertedAt: string
-  updatedAt: string
-}
-
-export interface UploadListOk extends ListResponse<UploadListItem> {}
-
 export interface ListOptions {
   size?: number
   cursor?: string
   pre?: boolean
 }
 
-export interface ListResponse<R> {
-  // cursor and after should be identical
-  cursor?: string
-  before?: string
-  after?: string
-  size: number
-  results: R[]
-}
-
 export interface TestSpaceRegistry {
   /**
    * Registers space with the registry.

diff --git a/packages/upload-api/src/upload/add.js b/packages/upload-api/src/upload/add.js
@@ -6,7 +6,7 @@ import { allocate } from '../space-allocate.js'
 
 /**
  * @param {API.UploadServiceContext} context
- * @returns {API.ServiceMethod<API.UploadAdd, API.UploadAddOk, API.Failure>}
+ * @returns {API.ServiceMethod<API.UploadAdd, API.UploadAddSuccess, API.Failure>}
  */
 export function uploadAddProvider(context) {
   return Server.provide(Upload.add, async ({ capability, invocation }) => {

diff --git a/packages/upload-api/src/upload/list.js b/packages/upload-api/src/upload/list.js
@@ -4,7 +4,7 @@ import * as API from '../types.js'
 
 /**
  * @param {API.UploadServiceContext} context
- * @returns {API.ServiceMethod<API.UploadList, API.UploadListOk, API.Failure>}
+ * @returns {API.ServiceMethod<API.UploadList, API.UploadListSuccess, API.Failure>}
  */
 export function uploadListProvider(context) {
   return Server.provide(Upload.list, async ({ capability }) => {

diff --git a/packages/upload-api/src/upload/remove.js b/packages/upload-api/src/upload/remove.js
@@ -4,7 +4,7 @@ import * as API from '../types.js'
 
 /**
  * @param {API.UploadServiceContext} context
- * @returns {API.ServiceMethod<API.UploadRemove, API.UploadRemoveOk, API.Failure>}
+ * @returns {API.ServiceMethod<API.UploadRemove, API.UploadRemoveSuccess, API.Failure>}
  */
 export function uploadRemoveProvider(context) {
   return Server.provide(Upload.remove, async ({ capability }) => {