feat(date-store-json): add JSON object storage implementation (#819)

* feat(data-store-json): add JSON backed storage option

* feat(data-store-json): add private-key-store

* feat(data-store-json): use the same JSON object backend for all the data stores

* feat(data-store-json): simplify JSON store APIs

convert integration test to use a plain file as the backend for the JSON storage.

* test(browser): use data-store-json in browser tests and unlock more test scenarios

* test(browser): link local packages during browser tests

* feat(core): define IDataStoreORM interface at the `@veramo/core` level

* docs(core): add some inline documentation for the `IDataStoreORM` plugin interface.

* docs(data-store-json): add some inline documentation for types defined in data-store-json

.gitignore

@@ -10,6 +10,8 @@
 /node_modules
 website/node_modules
 
+**/build/
+**/*.log
 /packages/*/build/
 /packages/*/api/*.api.json
 /packages/*/api/*.api.md

__browser_tests__/react-sample/headless-tests/browserAgent.test.js → __browser_tests__/react-sample/headless-tests/browserAgent.browser-test.ts

@@ -1,32 +1,32 @@
 import { agent } from '../src/veramo/setup'
 
 import keyManager from '../../../__tests__/shared/keyManager'
-import didManager from '../../../__tests__/shared/didManager'
+// import didManager from '../../../__tests__/shared/didManager'
 import verifiableDataJWT from '../../../__tests__/shared/verifiableDataJWT'
 import verifiableDataLD from '../../../__tests__/shared/verifiableDataLD'
 import handleSdrMessage from '../../../__tests__/shared/handleSdrMessage'
-import resolveDid from '../../../__tests__/shared/resolveDid'
+// import resolveDid from '../../../__tests__/shared/resolveDid'
 import webDidFlow from '../../../__tests__/shared/webDidFlow'
 import saveClaims from '../../../__tests__/shared/saveClaims'
 import documentationExamples from '../../../__tests__/shared/documentationExamples'
-import didCommPacking from '../../../__tests__/shared/didCommPacking'
+// import didCommPacking from '../../../__tests__/shared/didCommPacking'
 import messageHandler from '../../../__tests__/shared/messageHandler'
 
 jest.setTimeout(3 * 60 * 1000)
 
 describe('Browser integration tests', () => {
-  describe('global tests', () => {
+  describe('shared tests', () => {
     const testContext = { getAgent: () => agent, setup: () => {}, tearDown: () => {} }
-    // verifiableDataJWT(testContext)
+    verifiableDataJWT(testContext)
     // verifiableDataLD(testContext)
-    // handleSdrMessage(testContext)
+    handleSdrMessage(testContext)
     // resolveDid(testContext)
-    // webDidFlow(testContext)
-    // saveClaims(testContext)
-    // documentationExamples(testContext)
+    webDidFlow(testContext)
+    saveClaims(testContext)
+    documentationExamples(testContext)
     keyManager(testContext)
     // didManager(testContext)
-    // messageHandler(testContext)
+    messageHandler(testContext)
     // didCommPacking(testContext)
   })
 

__browser_tests__/react-sample/jest-integration.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   preset: 'jest-puppeteer',
   rootDir: 'headless-tests',
-  testRegex: './*\\.test\\.(js|tsx?)$',
+  testRegex: './*\\.browser-test\\.(ts|tsx?)$',
   globalSetup: 'jest-environment-puppeteer/setup',
   globalTeardown: 'jest-environment-puppeteer/teardown',
   testEnvironment: 'jest-environment-puppeteer',

__browser_tests__/react-sample/jest-puppeteer.config.js

@@ -6,7 +6,7 @@ module.exports = {
   },
   browserContext: "default",
   server: {
-    command: `npm start`,
+    command: "npm start",
     port: 3000,
     launchTimeout: 20000,
     debug: true,

__browser_tests__/react-sample/package.json

@@ -6,12 +6,26 @@
     "@types/node": "16.11.26",
     "@types/react": "17.0.39",
     "@types/react-dom": "17.0.13",
+    "@veramo/core": "^3.1.0",
+    "@veramo/credential-w3c": "^3.1.0",
+    "@veramo/data-store": "^3.1.0",
+    "@veramo/did-comm": "^3.1.0",
+    "@veramo/did-jwt": "^3.1.0",
+    "@veramo/did-manager": "^3.1.0",
+    "@veramo/did-provider-ethr": "^3.1.0",
+    "@veramo/did-provider-key": "^3.1.0",
+    "@veramo/did-provider-web": "^3.1.0",
+    "@veramo/did-resolver": "^3.1.0",
+    "@veramo/key-manager": "^3.1.0",
+    "@veramo/message-handler": "^3.1.0",
+    "@veramo/selective-disclosure": "^3.1.0",
     "crypto": "npm:crypto-browserify",
     "did-resolver": "3.1.5",
     "ethr-did-resolver": "5.0.4",
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-scripts": "5.0.0",
+    "typeorm": "^0.2.44",
     "typescript": "4.5.5",
     "web-did-resolver": "2.0.12",
     "web-vitals": "2.1.4"

__browser_tests__/react-sample/src/veramo/setup.ts

@@ -1,36 +1,40 @@
-import { createAgent, IResolver } from '@veramo/core'
+import { createAgent, IDataStore, IDataStoreORM, IDIDManager, IKeyManager, IResolver, TAgent } from '@veramo/core'
 
 import { DIDResolverPlugin } from '@veramo/did-resolver'
 import { Resolver } from 'did-resolver'
 import { getResolver as ethrDidResolver } from 'ethr-did-resolver'
 import { getResolver as webDidResolver } from 'web-did-resolver'
 import { MessageHandler } from '@veramo/message-handler'
-import { KeyManager, MemoryKeyStore, MemoryPrivateKeyStore } from '@veramo/key-manager'
-import { DIDManager, MemoryDIDStore } from '@veramo/did-manager'
+import { KeyManager } from '@veramo/key-manager'
+import { DIDManager } from '@veramo/did-manager'
 import { JwtMessageHandler } from '@veramo/did-jwt'
-import { W3cMessageHandler } from '@veramo/credential-w3c'
-import { DIDCommMessageHandler } from '@veramo/did-comm'
-import { SdrMessageHandler } from '@veramo/selective-disclosure'
+import { CredentialIssuer, ICredentialIssuer, W3cMessageHandler } from '@veramo/credential-w3c'
+// import { getDidKeyResolver, KeyDIDProvider } from '@veramo/did-provider-key'
+import { DIDComm, DIDCommMessageHandler, IDIDComm } from '@veramo/did-comm'
+import { ISelectiveDisclosure, SdrMessageHandler, SelectiveDisclosure } from '@veramo/selective-disclosure'
 import { KeyManagementSystem } from '@veramo/kms-local'
-// import { Connection, createConnection } from 'typeorm'
-import { CredentialIssuer, ICredentialIssuer } from '@veramo/credential-w3c'
-import {
-  CredentialIssuerLD,
-  ICredentialIssuerLD,
-  LdDefaultContexts,
-  VeramoEcdsaSecp256k1RecoverySignature2020,
-  VeramoEd25519Signature2018,
-} from '@veramo/credential-ld'
 import { EthrDIDProvider } from '@veramo/did-provider-ethr'
 import { WebDIDProvider } from '@veramo/did-provider-web'
-import { getDidKeyResolver, KeyDIDProvider } from '@veramo/did-provider-key'
-import { DIDComm, IDIDComm } from '@veramo/did-comm'
-import { ISelectiveDisclosure, SelectiveDisclosure } from '@veramo/selective-disclosure'
-// import { DataStore, DataStoreORM, Entities, IDataStoreORM, migrations } from '@veramo/data-store'
+import { DataStoreJson, DIDStoreJson, KeyStoreJson, PrivateKeyStoreJson } from "@veramo/data-store-json";
+// import { FakeDidProvider, FakeDidResolver } from "../../../../__tests__/utils/fake-did";
 
 const INFURA_PROJECT_ID = '33aab9e0334c44b0a2e0c57c15302608'
 
-export const agent = createAgent<IResolver>({
+const memoryJsonStore = {
+  notifyUpdate: () => Promise.resolve()
+}
+
+type InstalledPlugins =
+  IResolver
+  & IKeyManager
+  & IDIDManager
+  & ICredentialIssuer
+  & IDataStoreORM
+  & IDataStore
+  & ISelectiveDisclosure
+  & IDIDComm
+
+export const agent: TAgent<InstalledPlugins> = createAgent<InstalledPlugins>({
   plugins: [
     new DIDResolverPlugin({
       resolver: new Resolver({
@@ -47,13 +51,13 @@ export const agent = createAgent<IResolver>({
       }),
     }),
     new KeyManager({
-      store: new MemoryKeyStore(),
+      store: new KeyStoreJson(memoryJsonStore),
       kms: {
-        local: new KeyManagementSystem(new MemoryPrivateKeyStore()),
+        local: new KeyManagementSystem(new PrivateKeyStoreJson(memoryJsonStore)),
       },
     }),
     new DIDManager({
-      store: new MemoryDIDStore(),
+      store: new DIDStoreJson(memoryJsonStore),
       defaultProvider: 'did:ethr:rinkeby',
       providers: {
         'did:ethr': new EthrDIDProvider({
@@ -83,16 +87,15 @@ export const agent = createAgent<IResolver>({
          * `KeyDIDProvider` throws error:
          * "Field 'browser' doesn't contain a valid alias configuration"
          * Can't resolve 'stream'
-         * 'stream-browserify' can be installed for brower env
+         * 'stream-browserify' can be installed for browser env
          */
         // 'did:key': new KeyDIDProvider({
         //   defaultKms: 'local',
         // }),
         // 'did:fake': new FakeDidProvider(),
       },
     }),
-    // new DataStore(dbConnection),
-    // new DataStoreORM(dbConnection),
+    new DataStoreJson(memoryJsonStore),
     new MessageHandler({
       messageHandlers: [
         new DIDCommMessageHandler(),

__browser_tests__/react-sample/tsconfig.json

@@ -1,4 +1,5 @@
 {
+  "extends": "../../packages/tsconfig.settings.json",
   "compilerOptions": {
     "target": "es5",
     "lib": [
@@ -12,7 +13,6 @@
     "allowSyntheticDefaultImports": true,
     "strict": true,
     "forceConsistentCasingInFileNames": true,
-    "noFallthroughCasesInSwitch": true,
     "module": "esnext",
     "moduleResolution": "node",
     "resolveJsonModule": true,
@@ -22,5 +22,27 @@
   },
   "include": [
     "src"
+  ],
+  "references": [
+    { "path": "../../packages/core" },
+    { "path": "../../packages/credential-w3c" },
+    { "path": "../../packages/data-store" },
+    { "path": "../../packages/data-store-json" },
+    { "path": "../../packages/did-comm" },
+    { "path": "../../packages/did-discovery" },
+    { "path": "../../packages/did-jwt" },
+    { "path": "../../packages/did-manager" },
+    { "path": "../../packages/did-provider-ethr" },
+    { "path": "../../packages/did-provider-key" },
+    { "path": "../../packages/did-provider-web" },
+    { "path": "../../packages/did-resolver" },
+    { "path": "../../packages/key-manager" },
+    { "path": "../../packages/kms-local" },
+    { "path": "../../packages/message-handler" },
+    { "path": "../../packages/remote-client" },
+    { "path": "../../packages/remote-server" },
+    { "path": "../../packages/selective-disclosure" },
+    { "path": "../../packages/url-handler" },
+    { "path": "../../packages/utils" }
   ]
 }