feat: usage/record capability definition (#1562)

### Add `usage/record` Capability definition for Egress Traffic Tracking

**Summary:**
- Introduced a new capability `usage/record` definition to track egress
traffic metrics.
- The `w3upInfra` upload-api will be updated to implement this new
capability by leveraging Stripe's Usage Records API for accurate
tracking and billing.

RFC: storacha/RFC#37

packages/capabilities/src/types.ts

@@ -131,6 +131,10 @@ export type UsageReport = InferInvokedCapability<typeof UsageCaps.report>
 export type UsageReportSuccess = Record<ProviderDID, UsageData>
 export type UsageReportFailure = Ucanto.Failure
 
+export type EgressRecord = InferInvokedCapability<typeof UsageCaps.record>
+export type EgressRecordSuccess = Unit
+export type EgressRecordFailure = ConsumerNotFound | Ucanto.Failure
+
 export interface UsageData {
   /** Provider the report concerns, e.g. `did:web:web3.storage` */
   provider: ProviderDID
@@ -161,6 +165,21 @@ export interface UsageData {
   }>
 }
 
+export interface EgressData {
+  /** The space which contains the resource that was served. */
+  space: SpaceDID
+  /** The customer that is being billed for the egress traffic. */
+  customer: AccountDID
+  /** CID of the resource that was served it's the CID of some gateway accessible content. It is not the CID of a blob/shard.*/
+  resource: UnknownLink
+  /** Amount of bytes served. */
+  bytes: number
+  /** ISO datetime that the bytes were served at. */
+  servedAt: ISO8601Date
+  /** Identifier of the invocation that caused the egress traffic. */
+  cause: UnknownLink
+}
+
 // Provider
 export type ProviderAdd = InferInvokedCapability<typeof provider.add>
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
@@ -193,6 +212,7 @@ export interface ConsumerGetSuccess {
   allocated: number
   limit: number
   subscription: string
+  customer: AccountDID
 }
 export interface ConsumerNotFound extends Ucanto.Failure {
   name: 'ConsumerNotFound'

packages/capabilities/src/usage.js

@@ -40,3 +40,20 @@ export const report = capability({
     )
   },
 })
+
+/**
+ * Capability can be invoked by an agent to record usage data for a given resource.
+ */
+export const record = capability({
+  can: 'usage/record',
+  with: SpaceDID,
+  nb: Schema.struct({
+    /** CID of the resource that was served. */
+    resource: Schema.link(),
+    /** Amount of bytes served. */
+    bytes: Schema.integer().greaterThan(0),
+    /** Timestamp of the event in seconds after Unix epoch. */
+    servedAt: Schema.integer().greaterThan(-1),
+  }),
+  derives: equalWith,
+})

packages/upload-api/src/types/usage.ts

@@ -1,8 +1,10 @@
-import { Failure, Result } from '@ucanto/interface'
+import { Failure, Result, UnknownLink } from '@ucanto/interface'
 import {
+  AccountDID,
   ProviderDID,
   SpaceDID,
   UsageData,
+  EgressData,
 } from '@web3-storage/capabilities/types'
 
 export type { UsageData }
@@ -13,4 +15,18 @@ export interface UsageStorage {
     space: SpaceDID,
     period: { from: Date; to: Date }
   ) => Promise<Result<UsageData, Failure>>
+  record: (
+    /** The space which contains the resource that was served. */
+    space: SpaceDID,
+    /** The customer that is being billed for the egress traffic. */
+    customer: AccountDID,
+    /** The resource that was served to the customer through the gateway. */
+    resource: UnknownLink,
+    /** The number of bytes that were served. */
+    bytes: number,
+    /** The date and time when the resource was served. */
+    servedAt: Date,
+    /** Identifier of the invocation that caused the egress traffic. */
+    cause: UnknownLink,
+  ) => Promise<Result<EgressData, Failure>>
 }

packages/upload-api/src/usage.js

@@ -1,4 +1,8 @@
-import { provide } from './usage/report.js'
+import { provide as provideReport } from './usage/report.js'
+import { provide as provideRecord } from './usage/record.js'
 
 /** @param {import('./types.js').UsageServiceContext} context */
-export const createService = (context) => ({ report: provide(context) })
+export const createService = (context) => ({
+  report: provideReport(context),
+  record: provideRecord(context),
+})

packages/upload-api/src/usage/record.js

@@ -0,0 +1,41 @@
+import * as API from '../types.js'
+import * as Provider from '@ucanto/server'
+import { Usage } from '@web3-storage/capabilities'
+
+/** @param {API.UsageServiceContext} context */
+export const provide = (context) =>
+  Provider.provide(Usage.record, (input) => record(input, context))
+
+/**
+ * @param {API.Input<Usage.record>} input
+ * @param {API.UsageServiceContext} context
+ * @returns {Promise<API.Result<API.EgressRecordSuccess, API.EgressRecordFailure>>}
+ */
+const record = async ({ capability, invocation }, context) => {
+  const provider = /** @type {`did:web:${string}`} */ (
+    invocation.audience.did()
+  )
+  const consumerResponse = await context.provisionsStorage.getConsumer(
+    provider,
+    capability.with
+  )
+  if (consumerResponse.error) {
+    return consumerResponse
+  }
+  const consumer = consumerResponse.ok
+  const res = await context.usageStorage.record(
+    // The space which contains the resource that was served.
+    capability.with,
+    // The customer that is being billed for the egress traffic.
+    consumer.customer,
+    // CID of the resource that was served.
+    capability.nb.resource,
+    // Number of bytes that were served.
+    capability.nb.bytes,
+    // Date and time when the resource was served.
+    new Date(capability.nb.servedAt * 1000),
+    // Link to the invocation that caused the egress traffic.
+    invocation.cid
+  )
+  return res
+}

packages/upload-api/test/storage/provisions-storage.js

@@ -152,6 +152,7 @@ export class ProvisionsStorage {
           allocated: 0,
           limit: 100,
           subscription: itemKey(provision),
+          customer: provision.customer,
         },
       }
     } else {

packages/upload-api/test/storage/usage-storage.js

@@ -9,6 +9,10 @@ export class UsageStorage {
   constructor(storeTable, allocationsStorage) {
     this.storeTable = storeTable
     this.allocationsStorage = allocationsStorage
+    /**
+     * @type {Record<import('../types.js').AccountDID, import('../types.js').EgressData>}
+     */
+    this._egressRecords = {}
   }
 
   get items() {
@@ -64,4 +68,33 @@ export class UsageStorage {
       },
     }
   }
+
+  /**
+   * Simulate a record of egress data for a customer.
+   *
+   * @param {import('../types.js').SpaceDID} space
+   * @param {import('../types.js').AccountDID} customer
+   * @param {import('../types.js').UnknownLink} resource
+   * @param {number} bytes
+   * @param {Date} servedAt
+   * @param {import('../types.js').UnknownLink} cause
+   */
+  async record(space, customer, resource, bytes, servedAt, cause) {
+    const egressData = {
+      space,
+      customer,
+      resource,
+      bytes,
+      servedAt: servedAt.toISOString(),
+      cause,
+    }
+    this._egressRecords[customer] = egressData
+    return Promise.resolve({
+      ok: egressData,
+    })
+  }
+
+  get egressRecords() {
+    return this._egressRecords
+  }
 }

packages/upload-client/src/types.ts

@@ -69,6 +69,10 @@ import {
   UsageReport,
   UsageReportSuccess,
   UsageReportFailure,
+  EgressData,
+  EgressRecord,
+  EgressRecordSuccess,
+  EgressRecordFailure,
   ServiceAbility,
 } from '@web3-storage/capabilities/types'
 import { StorefrontService } from '@web3-storage/filecoin-client/storefront'
@@ -135,6 +139,10 @@ export type {
   UsageReport,
   UsageReportSuccess,
   UsageReportFailure,
+  EgressData,
+  EgressRecord,
+  EgressRecordSuccess,
+  EgressRecordFailure,
   ListResponse,
   CARLink,
   PieceLink,

packages/w3up-client/src/capability/usage.js

@@ -31,6 +31,37 @@ export class UsageClient extends Base {
 
     return out.ok
   }
+
+  /**
+   * Record egress data for a served resource.
+   * It will execute the capability invocation to find the customer and then record the egress data for the resource.
+   *
+   * Required delegated capabilities:
+   * - `usage/record`
+   *
+   * @param {import('../types.js').SpaceDID} space
+   * @param {object} egressData
+   * @param {API.UnknownLink} egressData.resource
+   * @param {number} egressData.bytes
+   * @param {string} egressData.servedAt
+   * @param {object} [options]
+   * @param {string} [options.nonce]
+   */
+  async record(space, egressData, options) {
+    const out = await record(
+      { agent: this.agent },
+      { space, ...egressData },
+      { ...options }
+    )
+    /* c8 ignore next 5 */
+    if (!out.ok) {
+      throw new Error(`failed ${UsageCapabilities.record.can} invocation`, {
+        cause: out.error,
+      })
+    }
+
+    return out.ok
+  }
 }
 
 /**
@@ -61,3 +92,35 @@ export const report = async (
   })
   return receipt.out
 }
+
+/**
+ * Record egress data for a resource from a given space.
+ *
+ * @param {{agent: API.Agent}} client
+ * @param {object} egressData
+ * @param {API.SpaceDID} egressData.space
+ * @param {API.UnknownLink} egressData.resource
+ * @param {number} egressData.bytes
+ * @param {string} egressData.servedAt
+ * @param {object} options
+ * @param {string} [options.nonce]
+ * @param {API.Delegation[]} [options.proofs]
+ * @returns {Promise<API.Result<API.Unit, API.EgressRecordFailure>>}
+ */
+export const record = async (
+  { agent },
+  { space, resource, bytes, servedAt },
+  { nonce, proofs = [] }
+) => {
+  const receipt = await agent.invokeAndExecute(UsageCapabilities.record, {
+    with: space,
+    proofs,
+    nonce,
+    nb: {
+      resource,
+      bytes,
+      servedAt: Math.floor(new Date(servedAt).getTime() / 1000),
+    },
+  })
+  return receipt.out
+}

packages/w3up-client/src/types.ts

@@ -141,6 +141,10 @@ export type {
   UploadListItem,
   UsageReportSuccess,
   UsageReportFailure,
+  EgressData,
+  EgressRecord,
+  EgressRecordSuccess,
+  EgressRecordFailure,
   ListResponse,
   AnyLink,
   CARLink,

packages/w3up-client/test/capability/usage.test.js

@@ -2,6 +2,7 @@ import { AgentData } from '@web3-storage/access/agent'
 import { Client } from '../../src/client.js'
 import * as Test from '../test.js'
 import { receiptsEndpoint } from '../helpers/utils.js'
+import { randomCAR } from '../helpers/random.js'
 
 export const UsageClient = Test.withContext({
   report: {
@@ -68,6 +69,46 @@ export const UsageClient = Test.withContext({
       assert.deepEqual(report, {})
     },
   },
+  record: {
+    'should record egress': async (
+      assert,
+      { connection, provisionsStorage }
+    ) => {
+      const alice = new Client(await AgentData.create(), {
+        // @ts-ignore
+        serviceConf: {
+          access: connection,
+          upload: connection,
+        },
+      })
+
+      const space = await alice.createSpace('test')
+      const auth = await space.createAuthorization(alice)
+      await alice.addSpace(auth)
+
+      // Then we setup a billing for this account
+      await provisionsStorage.put({
+        // @ts-expect-error
+        provider: connection.id.did(),
+        account: alice.agent.did(),
+        consumer: space.did(),
+      })
+
+      const car = await randomCAR(128)
+      const resource = car.cid
+      await alice.capability.upload.add(car.roots[0], [resource])
+
+      const result = await alice.capability.upload.get(car.roots[0])
+      assert.ok(result)
+
+      const record = await alice.capability.usage.record(space.did(), {
+        resource: resource.link(),
+        bytes: car.size,
+        servedAt: new Date().toISOString(),
+      })
+      assert.ok(record)
+    },
+  },
 })
 
 Test.test({ UsageClient })