extend build_proof for merkle multiproofs

This extends the `build_proof` helpers to use the new merkle multiproof implementation from the `nim-ssz-serialization` repo.
status-im · Nov 24, 2021 · e2a592a · e2a592a
1 parent 9c2f43e
commit e2a592a
Show file tree

Hide file tree

Showing 6 changed files with 140 additions and 77 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -211,9 +211,9 @@
 	url = https://github.com/status-im/nim-taskpools
 [submodule "vendor/nim-ssz-serialization"]
 	path = vendor/nim-ssz-serialization
-	url = https://github.com/status-im/nim-ssz-serialization.git
+	url = https://github.com/etan-status/nim-ssz-serialization.git
 	ignore = untracked
-	branch = master
+	branch = merkle-multiproof
 [submodule "vendor/nim-websock"]
 	path = vendor/nim-websock
 	url = https://github.com/status-im/nim-websock.git

diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim
@@ -88,7 +88,7 @@ iterator get_path_indices*(
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/ssz/merkle-proofs.md#merkle-multiproofs
 func get_helper_indices*(
-    indices: openArray[GeneralizedIndex]): seq[GeneralizedIndex] =
+    indices: varargs[GeneralizedIndex]): seq[GeneralizedIndex] =
   ## Get the generalized indices of all "extra" chunks in the tree needed
   ## to prove the chunks with the given generalized indices. Note that the
   ## decreasing order is chosen deliberately to ensure equivalence to the order
@@ -109,17 +109,20 @@ func get_helper_indices*(
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/ssz/merkle-proofs.md#merkle-multiproofs
 func check_multiproof_acceptable*(
-    indices: openArray[GeneralizedIndex]): Result[void, string] =
+    indices: varargs[GeneralizedIndex]): Result[void, string] =
   # Check that proof verification won't allocate excessive amounts of memory.
   const max_multiproof_complexity = nextPowerOfTwo(256)
   if indices.len > max_multiproof_complexity:
-    trace "Max multiproof complexity exceeded",
-      num_indices=indices.len, max_multiproof_complexity
+    when nimvm:
+      discard
+    else:
+      trace "Max multiproof complexity exceeded",
+        num_indices=indices.len, max_multiproof_complexity
     return err("Unsupported multiproof complexity (" & $indices.len & ")")
 
   if indices.len == 0:
     return err("No indices specified")
-  if indices.anyIt(it == 0.GeneralizedIndex):
+  if indices.anyIt(it <= 0.GeneralizedIndex):
     return err("Invalid index specified")
   ok()
 
@@ -246,8 +249,7 @@ func calculate_multi_merkle_root*(
   if leaves.len != indices.len:
     return err("Length mismatch for leaves and indices")
   ? check_multiproof_acceptable(indices)
-  calculate_multi_merkle_root_impl(
-    leaves, proof, indices, helper_indices)
+  calculate_multi_merkle_root_impl(leaves, proof, indices, helper_indices)
 
 func calculate_multi_merkle_root*(
     leaves: openArray[Eth2Digest],
@@ -256,8 +258,18 @@ func calculate_multi_merkle_root*(
   if leaves.len != indices.len:
     return err("Length mismatch for leaves and indices")
   ? check_multiproof_acceptable(indices)
-  calculate_multi_merkle_root_impl(
-    leaves, proof, indices, get_helper_indices(indices))
+  let helper_indices = get_helper_indices(indices)
+  calculate_multi_merkle_root_impl(leaves, proof, indices, helper_indices)
+
+func calculate_multi_merkle_root*(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: static openArray[GeneralizedIndex]): Result[Eth2Digest, string] =
+  if leaves.len != indices.len:
+    return err("Length mismatch for leaves and indices")
+  static: ? check_multiproof_acceptable(indices)
+  const helper_indices = get_helper_indices(indices)
+  calculate_multi_merkle_root_impl(leaves, proof, indices, helper_indices)
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs
 func verify_merkle_multiproof*(
@@ -279,6 +291,106 @@ func verify_merkle_multiproof*(
   if calc.isErr: return false
   calc.get == root
 
+func verify_merkle_multiproof*(
+    leaves: openArray[Eth2Digest],
+    proof: openArray[Eth2Digest],
+    indices: static openArray[GeneralizedIndex],
+    root: Eth2Digest): bool =
+  const helper_indices = get_helper_indices(indices)
+  let calc = calculate_multi_merkle_root(leaves, proof, indices, helper_indices)
+  if calc.isErr: return false
+  calc.get == root
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.3/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21
+func build_proof*(
+    anchor: auto,
+    indices: openArray[GeneralizedIndex],
+    helper_indices: openArray[GeneralizedIndex],
+    proof: var openArray[Eth2Digest]): Result[void, string] =
+  doAssert proof.len == helper_indices.len
+  ? check_multiproof_acceptable(indices)
+  hash_tree_root(anchor, helper_indices, proof)
+
+func build_proof*(
+    anchor: auto,
+    indices: openArray[GeneralizedIndex],
+    proof: var openArray[Eth2Digest]): Result[void, string] =
+  ? check_multiproof_acceptable(indices)
+  let helper_indices = get_helper_indices(indices)
+  doAssert proof.len == helper_indices.len
+  hash_tree_root(anchor, helper_indices, proof)
+
+func build_proof*(
+    anchor: auto,
+    indices: static openArray[GeneralizedIndex],
+    proof: var openArray[Eth2Digest]): Result[void, string] =
+  const v = check_multiproof_acceptable(indices)
+  when v.isErr:
+    result.err(v.error)
+  else:
+    const helper_indices = get_helper_indices(indices)
+    doAssert proof.len == helper_indices.len
+    hash_tree_root(anchor, helper_indices, proof)
+
+func build_proof*(
+    anchor: auto,
+    index: GeneralizedIndex,
+    proof: var openArray[Eth2Digest]): Result[void, string] =
+  ? check_multiproof_acceptable(index)
+  let helper_indices = get_helper_indices(index)
+  doAssert proof.len == helper_indices.len
+  hash_tree_root(anchor, helper_indices, proof)
+
+func build_proof*(
+    anchor: auto,
+    index: static GeneralizedIndex,
+    proof: var openArray[Eth2Digest]): Result[void, string] =
+  const v = check_multiproof_acceptable(index)
+  when v.isErr:
+    result.err(v.error)
+  else:
+    const helper_indices = get_helper_indices(index)
+    doAssert proof.len == helper_indices.len
+    hash_tree_root(anchor, helper_indices, proof)
+
+func build_proof*(
+    anchor: auto,
+    indices: openArray[GeneralizedIndex]
+): Result[seq[Eth2Digest], string] =
+  ? check_multiproof_acceptable(indices)
+  let helper_indices = get_helper_indices(indices)
+  hash_tree_root(anchor, helper_indices)
+
+func build_proof*(
+    anchor: auto,
+    indices: static openArray[GeneralizedIndex]
+): auto =
+  const v = check_multiproof_acceptable(indices)
+  when v.isErr:
+    Result[array[0, Eth2Digest], string].err(v.error)
+  else:
+    const helper_indices = get_helper_indices(indices)
+    hash_tree_root(anchor, helper_indices)
+
+func build_proof*(
+    anchor: auto,
+    index: GeneralizedIndex
+): Result[seq[Eth2Digest], string] =
+  ? check_multiproof_acceptable(index)
+  let helper_indices = get_helper_indices(index)
+  hash_tree_root(anchor, helper_indices)
+
+func build_proof*(
+    anchor: auto,
+    index: static GeneralizedIndex
+): auto =
+  const v = check_multiproof_acceptable(index)
+  when v.isErr:
+    Result[array[0, Eth2Digest], string].err(v.error)
+  else:
+    const helper_indices = get_helper_indices(index)
+    hash_tree_root(anchor, helper_indices)
+
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#is_valid_merkle_branch
 func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest],
                              depth: int, index: uint64,
@@ -299,59 +411,6 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest],
     value = eth2digest(buf)
   value == root
 
-# https://github.com/ethereum/consensus-specs/blob/v1.1.3/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21
-func build_proof_impl(anchor: object, leaf_index: uint64,
-                      proof: var openArray[Eth2Digest]) =
-  let
-    bottom_length = nextPow2(typeof(anchor).totalSerializedFields.uint64)
-    tree_depth = log2trunc(bottom_length)
-    parent_index =
-      if leaf_index < bottom_length shl 1:
-        0'u64
-      else:
-        var i = leaf_index
-        while i >= bottom_length shl 1:
-          i = i shr 1
-        i
-
-  var
-    prefix_len = 0
-    proof_len = log2trunc(leaf_index)
-    cache = newSeq[Eth2Digest](bottom_length shl 1)
-  block:
-    var i = bottom_length
-    anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar):
-      if i == parent_index:
-        when fieldVar is object:
-          prefix_len = log2trunc(leaf_index) - tree_depth
-          proof_len -= prefix_len
-          let
-            bottom_bits = leaf_index and not (uint64.high shl prefix_len)
-            prefix_leaf_index = (1'u64 shl prefix_len) + bottom_bits
-          build_proof_impl(fieldVar, prefix_leaf_index, proof)
-        else: raiseAssert "Invalid leaf_index"
-      cache[i] = hash_tree_root(fieldVar)
-      i += 1
-    for i in countdown(bottom_length - 1, 1):
-      cache[i] = withEth2Hash:
-        h.update cache[i shl 1].data
-        h.update cache[i shl 1 + 1].data
-
-  var i = if parent_index != 0: parent_index
-          else: leaf_index
-  doAssert i > 0 and i < bottom_length shl 1
-  for proof_index in prefix_len ..< prefix_len + proof_len:
-    let b = (i and 1) != 0
-    i = i shr 1
-    proof[proof_index] = if b: cache[i shl 1]
-                         else: cache[i shl 1 + 1]
-
-func build_proof*(anchor: object, leaf_index: uint64,
-                  proof: var openArray[Eth2Digest]) =
-  doAssert leaf_index > 0
-  doAssert proof.len == log2trunc(leaf_index)
-  build_proof_impl(anchor, leaf_index, proof)
-
 # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/altair/validator.md#sync-committee
 template sync_committee_period*(epoch: Epoch): SyncCommitteePeriod =
   (epoch div EPOCHS_PER_SYNC_COMMITTEE_PERIOD).SyncCommitteePeriod

diff --git a/tests/consensus_spec/altair/test_fixture_merkle_single_proof.nim b/tests/consensus_spec/altair/test_fixture_merkle_single_proof.nim
@@ -47,7 +47,7 @@ proc runTest(identifier: string) =
                                    altair.BeaconState))
 
       var computedProof = newSeq[Eth2Digest](log2trunc(proof.leaf_index))
-      build_proof(state[], proof.leaf_index, computedProof)
+      build_proof(state[], proof.leaf_index, computedProof).get
 
       check:
         computedProof == proof.branch.mapIt(Eth2Digest.fromHex(it))

diff --git a/tests/consensus_spec/altair/test_fixture_sync_protocol.nim b/tests/consensus_spec/altair/test_fixture_sync_protocol.nim
@@ -176,9 +176,8 @@ suite "Ethereum Foundation - Altair - Unittests - Sync protocol" & preset():
         forked[], committee, block_root = block_header.hash_tree_root())
 
     # Sync committee is updated
-    var next_sync_committee_branch {.noinit.}:
-      array[log2trunc(NEXT_SYNC_COMMITTEE_INDEX), Eth2Digest]
-    build_proof(state, NEXT_SYNC_COMMITTEE_INDEX, next_sync_committee_branch)
+    let next_sync_committee_branch =
+      build_proof(state, NEXT_SYNC_COMMITTEE_INDEX).get
     # Finality is unchanged
     let finality_header = BeaconBlockHeader()
     var finality_branch: array[log2trunc(FINALIZED_ROOT_INDEX), Eth2Digest]
@@ -246,9 +245,7 @@ suite "Ethereum Foundation - Altair - Unittests - Sync protocol" & preset():
         compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)
       finalized_block_header.hash_tree_root() ==
         state.finalized_checkpoint.root
-    var finality_branch {.noinit.}:
-      array[log2trunc(FINALIZED_ROOT_INDEX), Eth2Digest]
-    build_proof(state, FINALIZED_ROOT_INDEX, finality_branch)
+    let finality_branch = build_proof(state, FINALIZED_ROOT_INDEX).get
 
     # Build block header
     let

diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim
@@ -47,9 +47,11 @@ suite "Spec helpers":
       anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar):
         let depth = log2trunc(i)
         var proof = newSeq[Eth2Digest](depth)
-        build_proof(state, i, proof)
-        check: is_valid_merkle_branch(hash_tree_root(fieldVar), proof,
-                                      depth, get_subtree_index(i), root)
+        build_proof(state, i, proof).get
+        check:
+          hash_tree_root(fieldVar) == hash_tree_root(state, i).get
+          is_valid_merkle_branch(hash_tree_root(fieldVar), proof,
+                                 depth, get_subtree_index(i), root)
         when fieldVar is object and not (fieldVar is Eth2Digest):
           let
             numChildLeaves = fieldVar.numLeaves
@@ -96,9 +98,13 @@ suite "Spec helpers":
         ]
 
   test "verify_merkle_multiproof":
+    var allLeaves: array[8, Eth2Digest]
+    for i in 0 ..< allLeaves.len:
+      allLeaves[i] = eth2digest([i.byte])
+
     var nodes: array[16, Eth2Digest]
-    for i in countdown(15, 8):
-      nodes[i] = eth2digest([i.byte])
+    for i in 0 ..< allLeaves.len:
+      nodes[i + 8] = allLeaves[i]
     for i in countdown(7, 1):
       nodes[i] = withEth2Hash:
         h.update nodes[2 * i + 0].data
@@ -113,6 +119,7 @@ suite "Spec helpers":
         root = nodes[1]
       checkpoint "Verifying " & $indices & "---" & $helper_indices
       check:
+        proof == build_proof(allLeaves, indices).get
         verify_merkle_multiproof(leaves, proof, indices, root)
 
     verify([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])

diff --git a/vendor/nim-ssz-serialization b/vendor/nim-ssz-serialization