From 5ac4fe51a2a87a266e95c3db8215059ac2d38e77 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 28 Aug 2024 10:50:51 -0400 Subject: [PATCH] agent did api resolver cache (#855) In some cases, specifically when more than one identity exists, the `connect()` method can take over a minute to load. This is because DHT resolution takes places for each identity in the system when calling `identities.list()`. I won't go into the details of why resolution happens, but it's part of getting the signer to the DWN tenant for each identity. This PR implements @csuwildcat's cache implementation on the `AgentDidApi` class which takes into account which DIDs are managed by the agent (or the agent's own DID). These DIDs are never evicted from the cache, only updated versions are inserted after a successful resolution occurs. Other DIDs are evicted as usual. In addition to this I have increased the `recordId` reference storage cache to 21 days instead of 2 hours. These values don't change, so we should cache them for as long as possible. The max items is 1,000 and it will evict items based on last seen. 21 days was chosen because the maximum allowed timeout in Node is 24 days, I reduced it as a buffer. This should speed up `connect()` time and all signing of messages throughout the system. --- .changeset/kind-deers-burn.md | 9 ++ .github/workflows/alpha-npm.yml | 17 ++- audit-ci.json | 3 +- package.json | 3 +- packages/agent/package.json | 2 +- .../agent/src/agent-did-resolver-cache.ts | 72 +++++++++ packages/agent/src/did-api.ts | 19 ++- packages/agent/src/identity-api.ts | 13 +- packages/agent/src/index.ts | 1 + packages/agent/src/store-data.ts | 10 +- packages/agent/src/test-harness.ts | 5 +- .../tests/agent-did-resolver-cach.spec.ts | 142 ++++++++++++++++++ .../dids/src/resolver/resolver-cache-level.ts | 4 +- .../dids/src/resolver/universal-resolver.ts | 2 +- packages/user-agent/src/user-agent.ts | 3 +- pnpm-lock.yaml | 11 +- 16 files changed, 281 insertions(+), 35 deletions(-) create mode 100644 .changeset/kind-deers-burn.md create mode 100644 packages/agent/src/agent-did-resolver-cache.ts create mode 100644 packages/agent/tests/agent-did-resolver-cach.spec.ts diff --git a/.changeset/kind-deers-burn.md b/.changeset/kind-deers-burn.md new file mode 100644 index 000000000..f70eb6549 --- /dev/null +++ b/.changeset/kind-deers-burn.md @@ -0,0 +1,9 @@ +--- +"@web5/user-agent": patch +"@web5/agent": patch +"@web5/dids": patch +"@web5/identity-agent": patch +"@web5/proxy-agent": patch +--- + +Implement DidResolverCache thats specific to Agent usage diff --git a/.github/workflows/alpha-npm.yml b/.github/workflows/alpha-npm.yml index 1235f307c..a722ea5fd 100644 --- a/.github/workflows/alpha-npm.yml +++ b/.github/workflows/alpha-npm.yml @@ -21,7 +21,8 @@ jobs: env: # Packages not listed here will be excluded from publishing - PACKAGES: "agent api common credentials crypto crypto-aws-kms dids identity-agent proxy-agent user-agent" + # These are currently in a specific order due to dependency requirements + PACKAGES: "crypto crypto-aws-kms common dids credentials agent identity-agent proxy-agent user-agent api" steps: - name: Checkout source @@ -29,7 +30,7 @@ jobs: # https://cashapp.github.io/hermit/usage/ci/ - name: Init Hermit - uses: cashapp/activate-hermit@31ce88b17a84941bb1b782f1b7b317856addf286 #v1.1.0 + uses: cashapp/activate-hermit@v1 with: cache: "true" @@ -63,11 +64,17 @@ jobs: node ./scripts/bump-workspace.mjs --prerelease=$ALPHA_PRERELEASE shell: bash - - name: Build all workspace packages + - name: Build all workspace packages sequentially env: NODE_AUTH_TOKEN: ${{secrets.npm_token}} - run: pnpm --recursive --stream build - + run: | + for package in $PACKAGES; do + cd packages/$package + pnpm build + cd ../.. + done + shell: bash + - name: Publish selected @web5/* packages env: NODE_AUTH_TOKEN: ${{secrets.npm_token}} diff --git a/audit-ci.json b/audit-ci.json index 0311fe9e4..12d9d2333 100644 --- a/audit-ci.json +++ b/audit-ci.json @@ -6,6 +6,7 @@ "mysql2", "braces", "GHSA-rv95-896h-c2vc", - "GHSA-952p-6rrq-rcjv" + "GHSA-952p-6rrq-rcjv", + "GHSA-4vvj-4cpr-p986" ] } \ No newline at end of file diff --git a/package.json b/package.json index 12c283353..db1067aaa 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,8 @@ "elliptic@>=4.0.0 <=6.5.6": ">=6.5.7", "elliptic@>=2.0.0 <=6.5.6": ">=6.5.7", "elliptic@>=5.2.1 <=6.5.6": ">=6.5.7", - "micromatch@<4.0.8": ">=4.0.8" + "micromatch@<4.0.8": ">=4.0.8", + "webpack@<5.94.0": ">=5.94.0" } } } diff --git a/packages/agent/package.json b/packages/agent/package.json index eb67702d6..3c04ac5c2 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -74,7 +74,7 @@ "@tbd54566975/dwn-sdk-js": "0.4.6", "@web5/common": "1.0.0", "@web5/crypto": "workspace:*", - "@web5/dids": "1.1.0", + "@web5/dids": "workspace:*", "abstract-level": "1.0.4", "ed25519-keygen": "0.4.11", "isomorphic-ws": "^5.0.0", diff --git a/packages/agent/src/agent-did-resolver-cache.ts b/packages/agent/src/agent-did-resolver-cache.ts new file mode 100644 index 000000000..ecf35b76b --- /dev/null +++ b/packages/agent/src/agent-did-resolver-cache.ts @@ -0,0 +1,72 @@ +import { DidResolutionResult, DidResolverCache, DidResolverCacheLevel, DidResolverCacheLevelParams } from '@web5/dids'; +import { Web5PlatformAgent } from './types/agent.js'; + + +/** + * AgentDidResolverCache keeps a stale copy of the Agent's managed Identity DIDs and only evicts and refreshes upon a successful resolution. + * This allows for quick and offline access to the internal DIDs used by the agent. + */ +export class AgentDidResolverCache extends DidResolverCacheLevel implements DidResolverCache { + + /** + * Holds the instance of a `Web5PlatformAgent` that represents the current execution context for + * the `AgentDidApi`. This agent is used to interact with other Web5 agent components. It's vital + * to ensure this instance is set to correctly contextualize operations within the broader Web5 + * Agent framework. + */ + private _agent?: Web5PlatformAgent; + + /** A map of DIDs that are currently in-flight. This helps avoid going into an infinite loop */ + private _resolving: Map = new Map(); + + constructor({ agent, db, location, ttl }: DidResolverCacheLevelParams & { agent?: Web5PlatformAgent }) { + super ({ db, location, ttl }); + this._agent = agent; + } + + get agent() { + if (!this._agent) { + throw new Error('Agent not initialized'); + } + return this._agent; + } + + set agent(agent: Web5PlatformAgent) { + this._agent = agent; + } + + /** + * Get the DID resolution result from the cache for the given DID. + * + * If the DID is managed by the agent, or is the agent's own DID, it will not evict it from the cache until a new resolution is successful. + * This is done to achieve quick and offline access to the agent's own managed DIDs. + */ + async get(did: string): Promise { + try { + const str = await this.cache.get(did); + const cachedResult = JSON.parse(str); + if (!this._resolving.has(did) && Date.now() >= cachedResult.ttlMillis) { + this._resolving.set(did, true); + if (this.agent.agentDid.uri === did || 'undefined' !== typeof await this.agent.identity.get({ didUri: did })) { + try { + const result = await this.agent.did.resolve(did); + if (!result.didResolutionMetadata.error) { + this.set(did, result); + } + } finally { + this._resolving.delete(did); + } + } else { + this._resolving.delete(did); + this.cache.nextTick(() => this.cache.del(did)); + } + } + return cachedResult.value; + } catch(error: any) { + if (error.notFound) { + return; + } + throw error; + } + } +} \ No newline at end of file diff --git a/packages/agent/src/did-api.ts b/packages/agent/src/did-api.ts index 8b533b635..4a17d57af 100644 --- a/packages/agent/src/did-api.ts +++ b/packages/agent/src/did-api.ts @@ -3,12 +3,12 @@ import type { DidMetadata, PortableDid, DidMethodApi, - DidResolverCache, DidDhtCreateOptions, DidJwkCreateOptions, DidResolutionResult, DidResolutionOptions, DidVerificationMethod, + DidResolverCache, } from '@web5/dids'; import { BearerDid, Did, UniversalResolver } from '@web5/dids'; @@ -18,7 +18,7 @@ import type { AgentKeyManager } from './types/key-manager.js'; import type { ResponseStatus, Web5PlatformAgent } from './types/agent.js'; import { InMemoryDidStore } from './store-did.js'; -import { DidResolverCacheMemory } from './prototyping/dids/resolver-cache-memory.js'; +import { AgentDidResolverCache } from './agent-did-resolver-cache.js'; export enum DidInterface { Create = 'Create', @@ -87,8 +87,10 @@ export interface DidApiParams { * An optional `DidResolverCache` instance used for caching resolved DID documents. * * Providing a cache implementation can significantly enhance resolution performance by avoiding - * redundant resolutions for previously resolved DIDs. If omitted, a no-operation cache is used, - * which effectively disables caching. + * redundant resolutions for previously resolved DIDs. If omitted, the default is an instance of `AgentDidResolverCache`. + * + * `AgentDidResolverCache` keeps a stale copy of the Agent's managed Identity DIDs and only refreshes upon a successful resolution. + * This allows for quick and offline access to the internal DIDs used by the agent. */ resolverCache?: DidResolverCache; @@ -120,10 +122,10 @@ export class AgentDidApi } // Initialize the DID resolver with the given DID methods and resolver cache, or use a default - // in-memory cache if none is provided. + // AgentDidResolverCache if none is provided. super({ didResolvers : didMethods, - cache : resolverCache ?? new DidResolverCacheMemory() + cache : resolverCache ?? new AgentDidResolverCache({ agent, location: 'DATA/AGENT/DID_CACHE' }) }); this._agent = agent; @@ -152,6 +154,11 @@ export class AgentDidApi set agent(agent: Web5PlatformAgent) { this._agent = agent; + + // AgentDidResolverCache should set the agent if it is the type of cache being used + if ('agent' in this.cache) { + this.cache.agent = agent; + } } public async create({ diff --git a/packages/agent/src/identity-api.ts b/packages/agent/src/identity-api.ts index c64ef46ef..b857a67e6 100644 --- a/packages/agent/src/identity-api.ts +++ b/packages/agent/src/identity-api.ts @@ -183,14 +183,13 @@ export class AgentIdentityApi { + return this.get({ didUri: metadata.uri, tenant: metadata.tenant }); + }) + ); - for (const metadata of storedIdentities) { - const identity = await this.get({ didUri: metadata.uri, tenant: metadata.tenant }); - identities.push(identity!); - } - - return identities; + return identities.filter(identity => typeof identity !== 'undefined') as BearerIdentity[]; } public async manage({ portableIdentity }: { diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 5ed8caa62..7f7457575 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -7,6 +7,7 @@ export type * from './types/permissions.js'; export type * from './types/sync.js'; export type * from './types/vc.js'; +export * from './agent-did-resolver-cache.js'; export * from './bearer-identity.js'; export * from './cached-permissions.js'; export * from './crypto-api.js'; diff --git a/packages/agent/src/store-data.ts b/packages/agent/src/store-data.ts index 6873eaa0e..7d5699022 100644 --- a/packages/agent/src/store-data.ts +++ b/packages/agent/src/store-data.ts @@ -55,16 +55,20 @@ export class DwnDataStore = Jwk> implem /** * Index for mappings from Store Identifier to DWN record ID. + * Since these values don't change, we can use a long TTL. * - * Up to 1,000 entries are retained for 2 hours. + * Up to 1,000 entries are retained for 21 days. + * NOTE: The maximum number for the ttl is 2^31 - 1 milliseconds (24.8 days), setting to 21 days to be safe. */ - protected _index = new TtlCache({ ttl: ms('2 hours'), max: 1000 }); + protected _index = new TtlCache({ ttl: ms('21 days'), max: 1000 }); /** * Cache of tenant DIDs that have been initialized with the protocol. * This is used to avoid redundant protocol initialization requests. + * + * Since these are default protocols and unlikely to change, we can use a long TTL. */ - protected _protocolInitializedCache: TtlCache = new TtlCache({ ttl: ms('1 hour'), max: 1000 }); + protected _protocolInitializedCache: TtlCache = new TtlCache({ ttl: ms('21 days'), max: 1000 }); /** * The protocol assigned to this storage instance. diff --git a/packages/agent/src/test-harness.ts b/packages/agent/src/test-harness.ts index e17dc8df0..ec2e89d4d 100644 --- a/packages/agent/src/test-harness.ts +++ b/packages/agent/src/test-harness.ts @@ -4,11 +4,12 @@ import type { AbstractLevel } from 'abstract-level'; import { Level } from 'level'; import { LevelStore, MemoryStore } from '@web5/common'; import { DataStoreLevel, Dwn, EventEmitterStream, EventLogLevel, MessageStoreLevel, ResumableTaskStoreLevel } from '@tbd54566975/dwn-sdk-js'; -import { DidDht, DidJwk, DidResolutionResult, DidResolverCache, DidResolverCacheLevel } from '@web5/dids'; +import { DidDht, DidJwk, DidResolutionResult, DidResolverCache } from '@web5/dids'; import type { Web5PlatformAgent } from './types/agent.js'; import { AgentDidApi } from './did-api.js'; +import { AgentDidResolverCache } from './agent-did-resolver-cache.js'; import { AgentDwnApi } from './dwn-api.js'; import { AgentSyncApi } from './sync-api.js'; import { Web5RpcClient } from './rpc-client.js'; @@ -287,7 +288,7 @@ export class PlatformAgentTestHarness { const { didStore, identityStore, keyStore } = stores; // Setup DID Resolver Cache - const didResolverCache = new DidResolverCacheLevel({ + const didResolverCache = new AgentDidResolverCache({ location: testDataPath('DID_RESOLVERCACHE') }); diff --git a/packages/agent/tests/agent-did-resolver-cach.spec.ts b/packages/agent/tests/agent-did-resolver-cach.spec.ts new file mode 100644 index 000000000..37b7536b0 --- /dev/null +++ b/packages/agent/tests/agent-did-resolver-cach.spec.ts @@ -0,0 +1,142 @@ +import { AgentDidResolverCache } from '../src/agent-did-resolver-cache.js'; +import { PlatformAgentTestHarness } from '../src/test-harness.js'; +import { TestAgent } from './utils/test-agent.js'; + +import sinon from 'sinon'; +import { expect } from 'chai'; +import { DidJwk } from '@web5/dids'; +import { BearerIdentity } from '../src/bearer-identity.js'; + +describe('AgentDidResolverCache', () => { + let resolverCache: AgentDidResolverCache; + let testHarness: PlatformAgentTestHarness; + + before(async () => { + testHarness = await PlatformAgentTestHarness.setup({ + agentClass : TestAgent, + agentStores : 'dwn' + }); + + resolverCache = new AgentDidResolverCache({ agent: testHarness.agent, location: '__TESTDATA__/did_cache' }); + }); + + after(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.closeStorage(); + }); + + beforeEach(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.createAgentDid(); + }); + + it('does not attempt to resolve a DID that is already resolving', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() - 1000, value: { didDocument: { id: did } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + + await Promise.all([ + resolverCache.get(did), + resolverCache.get(did) + ]); + + // get should be called twice, but resolve should only be called once + // because the second call should be blocked by the _resolving Map + expect(getStub.callCount).to.equal(2); + expect(resolveSpy.callCount).to.equal(1); + }); + + it('should not resolve a DID if the ttl has not elapsed', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() + 1000, value: { didDocument: { id: did } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + + await resolverCache.get(did); + + // get should be called once, but resolve should not be called + expect(getStub.callCount).to.equal(1); + expect(resolveSpy.callCount).to.equal(0); + }); + + it('should not call resolve if the DID is not the agent DID or exists as an identity in the agent', async () => { + const did = await DidJwk.create({}); + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() - 1000, value: { didDocument: { id: did.uri } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + const nextTickSpy = sinon.stub(resolverCache['cache'], 'nextTick').resolves(); + sinon.stub(testHarness.agent.identity, 'get').resolves(undefined); + + await resolverCache.get(did.uri), + + // get should be called once, but we do not resolve even though the TTL is expired + expect(getStub.callCount).to.equal(1); + expect(resolveSpy.callCount).to.equal(0); + + // we expect the nextTick of the cache to be called to trigger a delete of the cache item after returning as it's expired + expect(nextTickSpy.callCount).to.equal(1); + }); + + it('should resolve if the DID is managed by the agent', async () => { + const did = await DidJwk.create({}); + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() - 1000, value: { didDocument: { id: did.uri } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + const nextTickSpy = sinon.stub(resolverCache['cache'], 'nextTick').resolves(); + sinon.stub(testHarness.agent.identity, 'get').resolves(new BearerIdentity({ + metadata: { name: 'Some Name', uri: did.uri, tenant: did.uri }, + did, + })); + + await resolverCache.get(did.uri), + + // get should be called once, and we also resolve the DId as it's returned by the identity.get method + expect(getStub.callCount).to.equal(1); + expect(resolveSpy.callCount).to.equal(1); + }); + + it('does not cache notFound records', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').rejects({ notFound: true }); + + const result = await resolverCache.get(did); + + // get should be called once, and resolve should be called once + expect(getStub.callCount).to.equal(1); + expect(result).to.equal(undefined); + }); + + it('throws if the error is anything other than a notFound error', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').rejects(new Error('Some Error')); + + try { + await resolverCache.get(did); + expect.fail('Should have thrown'); + } catch (error: any) { + expect(error.message).to.equal('Some Error'); + } + }); + + it('throws if the agent is not initialized', async () => { + // close existing DB + await resolverCache['cache'].close(); + + // set resolver cache without an agent + resolverCache = new AgentDidResolverCache({ location: '__TESTDATA__/did_cache' }); + + try { + // attempt to access the agent property + resolverCache.agent; + + expect.fail('Should have thrown'); + } catch (error: any) { + expect(error.message).to.equal('Agent not initialized'); + } + + // set the agent property + resolverCache.agent = testHarness.agent; + + // should not throw + resolverCache.agent; + }); +}); \ No newline at end of file diff --git a/packages/dids/src/resolver/resolver-cache-level.ts b/packages/dids/src/resolver/resolver-cache-level.ts index 69481e143..c4d2f2f80 100644 --- a/packages/dids/src/resolver/resolver-cache-level.ts +++ b/packages/dids/src/resolver/resolver-cache-level.ts @@ -73,10 +73,10 @@ type CachedDidResolutionResult = { */ export class DidResolverCacheLevel implements DidResolverCache { /** The underlying LevelDB store used for caching. */ - private cache; + protected cache; /** The time-to-live for cache entries in milliseconds. */ - private ttl: number; + protected ttl: number; constructor({ db, diff --git a/packages/dids/src/resolver/universal-resolver.ts b/packages/dids/src/resolver/universal-resolver.ts index 93ff11fb8..e938613ff 100644 --- a/packages/dids/src/resolver/universal-resolver.ts +++ b/packages/dids/src/resolver/universal-resolver.ts @@ -66,7 +66,7 @@ export class UniversalResolver implements DidResolver, DidUrlDereferencer { /** * A cache for storing resolved DID documents. */ - private cache: DidResolverCache; + protected cache: DidResolverCache; /** * A map to store method resolvers against method names. diff --git a/packages/user-agent/src/user-agent.ts b/packages/user-agent/src/user-agent.ts index 426a0668d..9568959fb 100644 --- a/packages/user-agent/src/user-agent.ts +++ b/packages/user-agent/src/user-agent.ts @@ -12,6 +12,7 @@ import { ProcessDwnRequest, Web5PlatformAgent, AgentPermissionsApi, + AgentDidResolverCache, } from '@web5/agent'; import { LevelStore } from '@web5/common'; @@ -152,7 +153,7 @@ export class Web5UserAgent=2.0.0 <=6.5.6: '>=6.5.7' elliptic@>=5.2.1 <=6.5.6: '>=6.5.7' micromatch@<4.0.8: '>=4.0.8' + webpack@<5.94.0: '>=5.94.0' importers: @@ -65,8 +66,8 @@ importers: specifier: workspace:* version: link:../crypto '@web5/dids': - specifier: 1.1.0 - version: 1.1.0 + specifier: workspace:* + version: link:../dids abstract-level: specifier: 1.0.4 version: 1.0.4 @@ -5323,13 +5324,13 @@ packages: resolution: {integrity: sha512-oYwAqCuL0OZhBoSgmdrLa7mv9MjommVMiQIWgcztf+eS4+8BfcUee6nenFnDhKOhzAVnk5gpZdfnz1iiBv+5sg==} engines: {node: '>= 14.15.0'} peerDependencies: - webpack: ^5.72.1 + webpack: '>=5.94.0' source-map-loader@5.0.0: resolution: {integrity: sha512-k2Dur7CbSLcAH73sBcIkV5xjPV4SzqO1NJ7+XaQl8if3VODDUj3FNchNGpqgJSKbvUfJuhVdv8K2Eu8/TNl2eA==} engines: {node: '>= 18.12.0'} peerDependencies: - webpack: ^5.72.1 + webpack: '>=5.94.0' source-map-support@0.5.21: resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} @@ -5507,7 +5508,7 @@ packages: '@swc/core': '*' esbuild: '*' uglify-js: '*' - webpack: ^5.1.0 + webpack: '>=5.94.0' peerDependenciesMeta: '@swc/core': optional: true