feat(key-manager): move private key storage to kms-local

fixes #539
fixes #540

BREAKING CHANGE: `keyManagetGet` no longer includes private key data
BREAKING CHANGE: `KeyStore` no longer requires a `SecretBox`
BREAKING CHANGE: `KeyManagementSystem` needs a `PrivateKeyStore`
BREAKING CHANGE: @veramo/cli configuration version update to 3.0

__tests__/localAgent.test.ts

@@ -34,6 +34,7 @@ import {
   DataStore,
   DataStoreORM,
   ProfileDiscoveryProvider,
+  PrivateKeyStore,
 } from '../packages/data-store/src'
 import { getDidKeyResolver } from '../packages/did-provider-key/src'
 import { IDIDDiscovery, DIDDiscovery } from '../packages/did-discovery/src'
@@ -105,9 +106,9 @@ const setup = async (options?: IAgentOptions): Promise<boolean> => {
     },
     plugins: [
       new KeyManager({
-        store: new KeyStore(dbConnection, new SecretBox(secretKey)),
+        store: new KeyStore(dbConnection),
         kms: {
-          local: new KeyManagementSystem(),
+          local: new KeyManagementSystem({ keyStore: new PrivateKeyStore(dbConnection, new SecretBox(secretKey)) }),
         },
       }),
       new DIDManager({

__tests__/localMemoryStoreAgent.test.ts

@@ -9,7 +9,7 @@ import {
   IAgentOptions,
 } from '../packages/core/src'
 import { MessageHandler } from '../packages/message-handler/src'
-import { KeyManager, MemoryKeyStore } from '../packages/key-manager/src'
+import { KeyManager, MemoryKeyStore, MemoryPrivateKeyStore } from '../packages/key-manager/src'
 import { DIDManager, MemoryDIDStore } from '../packages/did-manager/src'
 import { createConnection, Connection } from 'typeorm'
 import { DIDResolverPlugin } from '../packages/did-resolver/src'
@@ -93,7 +93,7 @@ const setup = async (options?: IAgentOptions): Promise<boolean> => {
       new KeyManager({
         store: new MemoryKeyStore(),
         kms: {
-          local: new KeyManagementSystem(),
+          local: new KeyManagementSystem({ keyStore: new MemoryPrivateKeyStore() }),
         },
       }),
       new DIDManager({

__tests__/restAgent.test.ts

@@ -37,6 +37,7 @@ import {
   DataStore,
   DataStoreORM,
   ProfileDiscoveryProvider,
+  PrivateKeyStore,
 } from '../packages/data-store/src'
 import { AgentRestClient } from '../packages/remote-client/src'
 import { AgentRouter, RequestWithAgentRouter, MessagingRouter } from '../packages/remote-server/src'
@@ -113,9 +114,9 @@ const setup = async (options?: IAgentOptions): Promise<boolean> => {
     ...options,
     plugins: [
       new KeyManager({
-        store: new KeyStore(dbConnection, new SecretBox(secretKey)),
+        store: new KeyStore(dbConnection),
         kms: {
-          local: new KeyManagementSystem(),
+          local: new KeyManagementSystem({ keyStore: new PrivateKeyStore(dbConnection, new SecretBox(secretKey)) }),
         },
       }),
       new DIDManager({

__tests__/shared/didManager.ts

@@ -261,56 +261,57 @@ export default (testContext: {
     })
 
     it('should import identifier', async () => {
-      const identifier = await agent.didManagerGetOrCreate({
-        alias: 'example.org',
+      expect.assertions(1)
+      const did = 'did:web:imported.example'
+      const imported = await agent.didManagerImport({
+        did,
         provider: 'did:web',
+        services: [
+          {
+            id: `${did}#msg`,
+            type: 'Messaging',
+            serviceEndpoint: 'https://example.org/messaging',
+            description: 'Handles incoming messages',
+          },
+        ],
+        keys: [
+          {
+            kms: 'local',
+            privateKeyHex: 'e63886b5ba367dc2aff9acea6d955ee7c39115f12eaf2aa6b1a2eaa852036668',
+            type: 'Secp256k1',
+          },
+        ],
+      })
+      expect(imported).toEqual({
+        did,
+        keys: [
+          {
+            kid: '04dd467afb12bdb797303e7f3f0c8cd0ba80d518dc4e339e0e2eb8f2d99a9415cac537854a30d31a854b7af0b4fcb54c3954047390fa9500d3cc2e15a3e09017bb',
+            kms: 'local',
+            meta: {
+              algorithms: [
+                'ES256K',
+                'ES256K-R',
+                'eth_signTransaction',
+                'eth_signTypedData',
+                'eth_signMessage',
+              ],
+            },
+            publicKeyHex:
+              '04dd467afb12bdb797303e7f3f0c8cd0ba80d518dc4e339e0e2eb8f2d99a9415cac537854a30d31a854b7af0b4fcb54c3954047390fa9500d3cc2e15a3e09017bb',
+            type: 'Secp256k1',
+          },
+        ],
+        provider: 'did:web',
+        services: [
+          {
+            description: 'Handles incoming messages',
+            id: `${did}#msg`,
+            serviceEndpoint: 'https://example.org/messaging',
+            type: 'Messaging',
+          } as Service,
+        ],
       })
-
-      await agent.didManagerAddService({
-        did: identifier.did,
-        service: {
-          id: 'did:web:example.org#msg',
-          type: 'Messaging',
-          serviceEndpoint: 'https://example.org/messaging',
-          description: 'Handles incoming messages',
-        },
-      })
-
-      const signingKeyFull = await agent.keyManagerGet({
-        kid: identifier.keys[0].kid,
-      })
-
-      const encryptionKey = await agent.keyManagerCreate({
-        kms: 'local',
-        type: 'Ed25519',
-      })
-
-      const encryptionKeyFull = await agent.keyManagerGet({
-        kid: encryptionKey.kid,
-      })
-
-      await agent.didManagerAddKey({
-        did: identifier.did,
-        key: encryptionKey,
-      })
-
-      const exportedIdentifier = await agent.didManagerGet({
-        did: identifier.did,
-      })
-
-      await agent.didManagerDelete({
-        did: identifier.did,
-      })
-
-      await agent.didManagerImport({
-        ...exportedIdentifier,
-        keys: [signingKeyFull, encryptionKeyFull],
-      })
-
-      const importedIdentifier = await agent.didManagerGet({
-        did: identifier.did,
-      })
-      expect(importedIdentifier).toEqual(exportedIdentifier)
     })
 
     it('should set alias for identifier', async () => {

__tests__/shared/keyManager.ts

@@ -69,7 +69,9 @@ export default (testContext: {
           kms: 'foobar',
           type: 'Secp256k1',
         }),
-      ).rejects.toThrow('KMS does not exist: foobar')
+      ).rejects.toThrow(
+        `invalid_argument: This agent has no registered KeyManagementSystem with name='foobar'`,
+      )
     })
 
     it('should throw an error for unsupported key type', async () => {
@@ -115,8 +117,12 @@ export default (testContext: {
         kid: key.kid,
       })
 
-      expect(key2).toHaveProperty('privateKeyHex')
-      expect(key2.publicKeyHex).toEqual(key.publicKeyHex)
+      expect(key2).toHaveProperty('kid')
+      expect(key2).toHaveProperty('kms')
+      expect(key2).toHaveProperty('publicKeyHex')
+      expect(key2).toHaveProperty('type')
+      expect(key2).not.toHaveProperty('privateKeyHex')
+      expect(key2).toEqual(key)
     })
 
     it('should delete key', async () => {
@@ -145,24 +151,34 @@ export default (testContext: {
     })
 
     it('should import key', async () => {
-      const fullKey: IKey = {
-        kid: '04dd467afb12bdb797303e7f3f0c8cd0ba80d518dc4e339e0e2eb8f2d99a9415cac537854a30d31a854b7af0b4fcb54c3954047390fa9500d3cc2e15a3e09017bb',
+      const keyData = {
+        kid: 'myImportedKey',
+        kms: 'local',
+        type: <TKeyType>'Secp256k1',
+        privateKeyHex: 'e63886b5ba367dc2aff9acea6d955ee7c39115f12eaf2aa6b1a2eaa852036668',
+        meta: { foo: 'bar' },
+      }
+
+      const expectedImport = {
+        kid: 'myImportedKey',
         kms: 'local',
         type: 'Secp256k1',
         publicKeyHex:
           '04dd467afb12bdb797303e7f3f0c8cd0ba80d518dc4e339e0e2eb8f2d99a9415cac537854a30d31a854b7af0b4fcb54c3954047390fa9500d3cc2e15a3e09017bb',
-        privateKeyHex: 'e63886b5ba367dc2aff9acea6d955ee7c39115f12eaf2aa6b1a2eaa852036668',
-        meta: { foo: 'bar' },
+        meta: {
+          algorithms: ['ES256K', 'ES256K-R', 'eth_signTransaction', 'eth_signTypedData', 'eth_signMessage'],
+          foo: 'bar',
+        },
       }
 
-      const result = await agent.keyManagerImport(fullKey)
-      expect(result).toEqual(true)
+      const result = await agent.keyManagerImport(keyData)
+      expect(result).toEqual(expectedImport)
 
       const key2 = await agent.keyManagerGet({
-        kid: fullKey.kid,
+        kid: keyData.kid,
       })
 
-      expect(key2).toEqual(fullKey)
+      expect(key2).toEqual(expectedImport)
     })
 
     it('should sign JWT', async () => {

package.json

@@ -10,7 +10,7 @@
     "test:integration-prepare": "ts-node --project packages/tsconfig.settings.json ./scripts/prepare-integration-tests.ts",
     "test:integration-pretty": "prettier --write __tests__/shared/documentationExamples.ts",
     "test:integration": "yarn test:integration-build && yarn test:ci",
-    "test:ci": "jest --config=jest.json --maxWorkers=2",
+    "test:ci": "jest --config=jest.json",
     "test": "jest --config=jest.json --coverage=false",
     "test:watch": "yarn test --watch --verbose",
     "veramo": "./packages/cli/bin/veramo.js",

packages/cli/default/default.yml

@@ -1,4 +1,4 @@
-version: 2.0
+version: 3.0
 
 constants:
   baseUrl: http://localhost:3332
@@ -173,20 +173,7 @@ didResolver:
 ethr-did-resolver:
   $require: ethr-did-resolver?t=function&p=/ethr#getResolver
   $args:
-    - networks:
-        - name: mainnet
-          rpcUrl: https://mainnet.infura.io/v3/5ffc47f65c4042ce847ef66a3fa70d4c
-        - name: rinkeby
-          rpcUrl: https://rinkeby.infura.io/v3/5ffc47f65c4042ce847ef66a3fa70d4c
-        - name: ropsten
-          rpcUrl: https://ropsten.infura.io/v3/5ffc47f65c4042ce847ef66a3fa70d4c
-        - name: kovan
-          rpcUrl: https://kovan.infura.io/v3/5ffc47f65c4042ce847ef66a3fa70d4c
-        - name: goerli
-          rpcUrl: https://goerli.infura.io/v3/5ffc47f65c4042ce847ef66a3fa70d4c
-        - name: private
-          rpcUrl: http://localhost:8545/
-          registry: '0x05cc574b19a3c11308f761b3d7263bd8608bc532'
+    - infuraProjectId: 5ffc47f65c4042ce847ef66a3fa70d4c
 
 web-did-resolver:
   $require: web-did-resolver?t=function&p=/web#getResolver
@@ -207,12 +194,17 @@ keyManager:
         $require: '@veramo/data-store#KeyStore'
         $args:
           - $ref: /dbConnection
-          - $require: '@veramo/kms-local#SecretBox'
-            $args:
-              - $ref: /constants/secretKey
       kms:
         local:
           $require: '@veramo/kms-local#KeyManagementSystem'
+          $args:
+            - keyStore:
+                $require: '@veramo/data-store#PrivateKeyStore'
+                $args:
+                  - $ref: /dbConnection
+                  - $require: '@veramo/kms-local#SecretBox'
+                    $args:
+                      - $ref: /constants/secretKey
 
 # DID Manager
 didManager:

packages/cli/src/migrations/SecretBox1588075773000.ts

@@ -9,8 +9,8 @@ export class SecretBox1588075773000 implements MigrationInterface {
       const secretBox = new SecretBox(process.env.VERAMO_SECRET_KEY)
       const keys = await queryRunner.connection.getRepository(Key).find()
       for (const key of keys) {
-        if (key.privateKeyHex) {
-          key.privateKeyHex = await secretBox.encrypt(key.privateKeyHex)
+        if ((<any>key).privateKeyHex) {
+          (<any>key).privateKeyHex = await secretBox.encrypt((<any>key).privateKeyHex)
         }
         await key.save()
       }
@@ -23,8 +23,8 @@ export class SecretBox1588075773000 implements MigrationInterface {
       const secretBox = new SecretBox(process.env.VERAMO_SECRET_KEY)
       const keys = await queryRunner.connection.getRepository(Key).find()
       for (const key of keys) {
-        if (key.privateKeyHex) {
-          key.privateKeyHex = await secretBox.decrypt(key.privateKeyHex)
+        if ((<any>key).privateKeyHex) {
+          (<any>key).privateKeyHex = await secretBox.decrypt((<any>key).privateKeyHex)
         }
         await key.save()
       }

packages/cli/src/setup.ts

@@ -19,7 +19,7 @@ export const getConfig = (fileName: string): any => {
 
   const config = yaml.parse(fs.readFileSync(fileName).toString())
 
-  if (config?.version != 2) {
+  if (config?.version != 3) {
     console.log('Unsupported configuration file version:', config.version)
     process.exit(1)
   }