feat(credentials): store and processing generic app credentials

tests/all_tests_v2.nim

@@ -27,7 +27,6 @@ when defined(waku_exp_store_resume):
   # TODO: Review store resume test cases (#1282)
   import ./v2/waku_store/test_resume
 
-
 import
   # Waku v2 tests
   ./v2/test_wakunode,
@@ -62,11 +61,12 @@ import
   ./v2/test_waku_noise,
   ./v2/test_waku_noise_sessions,
   ./v2/test_waku_switch,
+  # Waku Keystore
+  ./v2/test_waku_keystore_keyfile,
+  ./v2/test_waku_keystore,
   # Utils
-  ./v2/test_utils_compat,
-  ./v2/test_utils_keyfile
-
-
+  ./v2/test_utils_compat
+
 ## Experimental
 
 when defined(rln):

tests/v2/test_waku_keystore.nim

@@ -0,0 +1,191 @@
+{.used.}
+
+import
+  std/[algorithm, json, options, os],
+  testutils/unittests, chronos, stint,
+  ../../waku/v2/protocol/waku_keystore,
+  ../test_helpers
+
+from  ../../waku/v2/protocol/waku_noise/noise_utils import randomSeqByte
+
+procSuite "Credentials test suite":
+
+  # We initialize the RNG in test_helpers
+  let rng = rng()
+  let testAppInfo = AppInfo(application: "test", appIdentifier: "1234", version: "0.1")
+
+  asyncTest "Create keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    let keystoreRes = createAppKeystore(path = filepath,
+                                        appInfo = testAppInfo)
+
+    check:
+      keystoreRes.isOk()
+
+  asyncTest "Load keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    # If no keystore exists at filepath, a new one is created for appInfo and empty credentials
+    let keystoreRes = loadAppKeystore(path = filepath,
+                                      appInfo = testAppInfo)
+
+    check:
+      keystoreRes.isOk()
+
+    let keystore = keystoreRes.get()
+
+    check:
+      keystore.hasKeys(["application", "appIdentifier", "version", "credentials"])
+      keystore["application"].getStr() == testAppInfo.application
+      keystore["appIdentifier"].getStr() == testAppInfo.appIdentifier
+      keystore["version"].getStr() == testAppInfo.version
+      # We assume the loaded keystore to not have credentials set (previous tests delete the keystore at filepath)
+      keystore["credentials"].getElems().len() == 0
+
+  asyncTest "Add credentials to keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    # We generate a random identity credential (inter-value constrains are not enforced, otherwise we need to load e.g. zerokit RLN keygen)
+    var
+      idTrapdoor = randomSeqByte(rng[], 32)
+      idNullifier =  randomSeqByte(rng[], 32)
+      idSecretHash = randomSeqByte(rng[], 32)
+      idCommitment = randomSeqByte(rng[], 32)
+
+    var idCredential = IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, idCommitment: idCommitment)
+
+    var contract = MembershipContract(chainId: "5", address: "0x0123456789012345678901234567890123456789")
+    var index1 = MembershipIndex(1)
+    var membershipGroup1 = MembershipGroup(membershipContract: contract, treeIndex: index1)
+
+    let membershipCredentials1 = MembershipCredentials(identityCredential: idCredential,
+                                                       membershipGroups: @[membershipGroup1])
+
+    # We generate a random identity credential (inter-value constrains are not enforced, otherwise we need to load e.g. zerokit RLN keygen)
+    idTrapdoor = randomSeqByte(rng[], 32)
+    idNullifier =  randomSeqByte(rng[], 32)
+    idSecretHash = randomSeqByte(rng[], 32)
+    idCommitment = randomSeqByte(rng[], 32)
+
+    idCredential = IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, idCommitment: idCommitment)
+
+    var index2 = MembershipIndex(2)
+    var membershipGroup2 = MembershipGroup(membershipContract: contract, treeIndex: index2)
+
+    let membershipCredentials2 = MembershipCredentials(identityCredential: idCredential,
+                                                       membershipGroups: @[membershipGroup2])
+
+    let password = "%m0um0ucoW%"
+
+    let keystoreRes = addMembershipCredentials(path = filepath,
+                                               credentials = @[membershipCredentials1, membershipCredentials2],
+                                               password = password,
+                                               appInfo = testAppInfo)
+
+    check:
+      keystoreRes.isOk()
+
+  asyncTest "Add/retrieve credentials in keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    # We generate two random identity credentials (inter-value constrains are not enforced, otherwise we need to load e.g. zerokit RLN keygen)
+    var
+      idTrapdoor1 = randomSeqByte(rng[], 32)
+      idNullifier1 =  randomSeqByte(rng[], 32)
+      idSecretHash1 = randomSeqByte(rng[], 32)
+      idCommitment1 = randomSeqByte(rng[], 32)
+      idCredential1 = IdentityCredential(idTrapdoor: idTrapdoor1, idNullifier: idNullifier1, idSecretHash: idSecretHash1, idCommitment: idCommitment1)
+
+    var
+      idTrapdoor2 = randomSeqByte(rng[], 32)
+      idNullifier2 =  randomSeqByte(rng[], 32)
+      idSecretHash2 = randomSeqByte(rng[], 32)
+      idCommitment2 = randomSeqByte(rng[], 32)
+      idCredential2 = IdentityCredential(idTrapdoor: idTrapdoor2, idNullifier: idNullifier2, idSecretHash: idSecretHash2, idCommitment: idCommitment2)
+
+    # We generate two distinct membership groups
+    var contract1 = MembershipContract(chainId: "5", address: "0x0123456789012345678901234567890123456789")
+    var index1 = MembershipIndex(1)
+    var membershipGroup1 = MembershipGroup(membershipContract: contract1, treeIndex: index1)
+
+    var contract2 = MembershipContract(chainId: "6", address: "0x0000000000000000000000000000000000000000")
+    var index2 = MembershipIndex(2)
+    var membershipGroup2 = MembershipGroup(membershipContract: contract2, treeIndex: index2)
+
+    # We generate three membership credentials
+    let membershipCredentials1 = MembershipCredentials(identityCredential: idCredential1,
+                                                       membershipGroups: @[membershipGroup1])
+
+    let membershipCredentials2 = MembershipCredentials(identityCredential: idCredential2,
+                                                       membershipGroups: @[membershipGroup2])
+
+    let membershipCredentials3 = MembershipCredentials(identityCredential: idCredential1,
+                                                       membershipGroups: @[membershipGroup2])
+
+    # This is the same as rlnMembershipCredentials3, should not change the keystore entry of idCredential
+    let membershipCredentials4 = MembershipCredentials(identityCredential: idCredential1,
+                                                       membershipGroups: @[membershipGroup2])
+
+    let password = "%m0um0ucoW%"
+
+    # We add credentials to the keystore. Note that only 3 credentials should be effectively added, since rlnMembershipCredentials3 is equal to membershipCredentials2
+    let keystoreRes = addMembershipCredentials(path = filepath,
+                                               credentials = @[membershipCredentials1, membershipCredentials2, membershipCredentials3, membershipCredentials4],
+                                               password = password,
+                                               appInfo = testAppInfo)
+
+    check:
+      keystoreRes.isOk()
+
+    # We test retrieval of credentials.
+    var expectedMembershipGroups1 = @[membershipGroup1, membershipGroup2]          
+    expectedMembershipGroups1.sort(sortMembershipGroup)
+    let expectedCredential1 = MembershipCredentials(identityCredential: idCredential1,
+                                                    membershipGroups: expectedMembershipGroups1)
+
+
+    var expectedMembershipGroups2 = @[membershipGroup2]          
+    expectedMembershipGroups2.sort(sortMembershipGroup)
+    let expectedCredential2 = MembershipCredentials(identityCredential: idCredential2,
+                                                    membershipGroups: expectedMembershipGroups2)
+
+
+    # We retrieve all credentials stored under password (no filter)
+    var recoveredCredentialsRes = getMembershipCredentials(path = filepath,
+                                                           password = password,
+                                                           appInfo = testAppInfo)
+
+    check:
+      recoveredCredentialsRes.isOk()
+      recoveredCredentialsRes.get() == @[expectedCredential1, expectedCredential2]
+
+
+    # We retrieve credentials by filtering on an IdentityCredential
+    recoveredCredentialsRes = getMembershipCredentials(path = filepath,
+                                                       password = password,
+                                                       filterIdentityCredentials = @[idCredential1],
+                                                       appInfo = testAppInfo)
+
+    check:
+      recoveredCredentialsRes.isOk()
+      recoveredCredentialsRes.get() == @[expectedCredential1]
+
+    # We retrieve credentials by filtering on multiple IdentityCredentials
+    recoveredCredentialsRes = getMembershipCredentials(path = filepath,
+                                                       password = password,
+                                                       filterIdentityCredentials = @[idCredential1, idCredential2],
+                                                       appInfo = testAppInfo)
+
+    check:  
+      recoveredCredentialsRes.isOk()
+      recoveredCredentialsRes.get() == @[expectedCredential1, expectedCredential2]
+

tests/v2/test_utils_keyfile.nim → tests/v2/test_waku_keystore_keyfile.nim

@@ -6,7 +6,7 @@ import
   testutils/unittests, chronos,
   eth/keys
 import
-  ../../waku/v2/utils/keyfile
+  ../../waku/v2/protocol/waku_keystore
 
 from ../../waku/v2/protocol/waku_noise/noise_utils import randomSeqByte
 
@@ -312,7 +312,7 @@ suite "KeyFile test suite (adapted from nim-eth keyfile tests)":
 
       check:
         secret.isErr()
-        secret.error == KeyFileError.IncorrectMac
+        secret.error == KeyFileError.KeyfileIncorrectMac
 
   test "Wrong mac in keyfile":
 
@@ -350,7 +350,7 @@ suite "KeyFile test suite (adapted from nim-eth keyfile tests)":
                         keyfileWrongMac.getOrDefault("password").getStr())
     check:
       secret.isErr()
-      secret.error == KeyFileError.IncorrectMac
+      secret.error == KeyFileError.KeyFileIncorrectMac
 
   test "Scrypt keyfiles":
     let

tests/v2/test_waku_rln_relay.nim

@@ -14,12 +14,14 @@ import
   ../../waku/v2/node/waku_node,
   ../../waku/v2/protocol/waku_message,
   ../../waku/v2/protocol/waku_rln_relay,
+  ../../waku/v2/protocol/waku_keystore,
   ../test_helpers
 
 const RlnRelayPubsubTopic = "waku/2/rlnrelay/proto"
 const RlnRelayContentTopic = "waku/2/rlnrelay/proto"
 
 procSuite "Waku rln relay":
+
   asyncTest "mount waku-rln-relay in the off-chain mode":
     let
       nodeKey = crypto.PrivateKey.random(Secp256k1, rng[])[]
@@ -1041,10 +1043,11 @@ suite "Waku rln relay":
 
     debug "the generated identity credential: ", idCredential
 
-    let index =  MembershipIndex(1)
+    let index = MembershipIndex(1)
 
-    let rlnMembershipCredentials = RlnMembershipCredentials(identityCredential: idCredential, 
-                                                            rlnIndex: index)
+    let rlnMembershipContract = MembershipContract(chainId: "5", address: "0x0123456789012345678901234567890123456789")
+    let rlnMembershipGroup = MembershipGroup(membershipContract: rlnMembershipContract, treeIndex: index)
+    let rlnMembershipCredentials = MembershipCredentials(identityCredential: idCredential, membershipGroups: @[rlnMembershipGroup])
 
     let password = "%m0um0ucoW%"
 
@@ -1053,19 +1056,31 @@ suite "Waku rln relay":
 
     # Write RLN credentials
     require:
-      writeRlnCredentials(filepath, rlnMembershipCredentials, password).isOk()
+      addMembershipCredentials(path = filepath,
+                                credentials = @[rlnMembershipCredentials],
+                                password = password,
+                                appInfo = RLNAppInfo).isOk()
+
+    let readCredentialsResult = getMembershipCredentials(path = filepath,
+                                                         password = password,
+                                                         filterMembershipContracts = @[rlnMembershipContract],
+                                                         appInfo = RLNAppInfo)
 
-    let readCredentialsResult = readRlnCredentials(filepath, password)
     require:
       readCredentialsResult.isOk()
 
-    let credentials = readCredentialsResult.get()
+    # getMembershipCredentials returns all credentials in keystore as sequence matching the filter
+    let allMatchingCredentials = readCredentialsResult.get()
+    # if any is found, we return the first credential, otherwise credentials is none 
+    var credentials = none(MembershipCredentials)
+    if allMatchingCredentials.len() > 0:
+      credentials = some(allMatchingCredentials[0])
 
     require:
       credentials.isSome()
     check:
       credentials.get().identityCredential == idCredential
-      credentials.get().rlnIndex == index
+      credentials.get().membershipGroups == @[rlnMembershipGroup]
 
   test "histogram static bucket generation":
     let buckets = generateBucketsForHistogram(10)

tests/v2/test_waku_rln_relay_onchain.nim

@@ -8,6 +8,7 @@ import
   stew/byteutils, stew/shims/net as stewNet,
   libp2p/crypto/crypto,
   eth/keys,
+  ../../waku/v2/protocol/waku_keystore,
   ../../waku/v2/protocol/waku_rln_relay,
   ../../waku/v2/node/waku_node,
   ../test_helpers,

tests/v2/test_wakunode_rln_relay.nim

@@ -17,6 +17,7 @@ import
   ../../waku/v2/node/waku_node,
   ../../waku/v2/protocol/waku_message,
   ../../waku/v2/protocol/waku_rln_relay,
+  ../../waku/v2/protocol/waku_keystore,
   ../../waku/v2/utils/peers
 
 from std/times import epochTime

waku/v2/node/waku_node.nim

@@ -390,7 +390,7 @@ proc info*(node: WakuNode): WakuInfo =
 proc connectToNodes*(node: WakuNode, nodes: seq[RemotePeerInfo] | seq[string], source = "api") {.async.} =
   ## `source` indicates source of node addrs (static config, api call, discovery, etc)
   # NOTE This is dialing on WakuRelay protocol specifically
-  await connectToNodes(node.peerManager, nodes, WakuRelayCodec, source=source)
+  await peer_manager.connectToNodes(node.peerManager, nodes, WakuRelayCodec, source=source)
 
 
 ## Waku relay

waku/v2/protocol/waku_keystore.nim

@@ -0,0 +1,19 @@
+# The keyfile submodule (implementation adapted from nim-eth keyfile module https://github.com/status-im/nim-eth/blob/master/eth/keyfile)
+import
+  ./waku_keystore/keyfile
+
+export
+  keyfile
+
+# The Waku Keystore implementation
+import
+  ./waku_keystore/keystore,
+  ./waku_keystore/conversion_utils,
+  ./waku_keystore/protocol_types,
+  ./waku_keystore/utils
+
+export
+  keystore,
+  conversion_utils,
+  protocol_types,
+  utils

waku/v2/protocol/waku_keystore/conversion_utils.nim

@@ -0,0 +1,29 @@
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import 
+  json,
+  stew/[results, byteutils],
+  ./protocol_types
+
+# Encodes a Membership credential to a byte sequence
+proc encode*(credential: MembershipCredentials): seq[byte] =
+  # TODO: use custom encoding, avoid wordy json
+  var stringCredential: string
+  # NOTE: toUgly appends to the string, doesn't replace its contents
+  stringCredential.toUgly(%credential)
+  return toBytes(stringCredential)
+
+# Decodes a byte sequence to a Membership credential
+proc decode*(encodedCredential: seq[byte]): KeystoreResult[MembershipCredentials] =
+  # TODO: use custom decoding, avoid wordy json
+  try:
+    # we parse the json decrypted keystoreCredential
+    let jsonObject = parseJson(string.fromBytes(encodedCredential))
+    return ok(to(jsonObject, MembershipCredentials))
+  except JsonParsingError:
+    return err(KeystoreJsonError)
+  except Exception: #parseJson raises Exception
+    return err(KeystoreOsError)