From e2a592a086a5be6f7ed990077cfed840b3327c0b Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 25 Oct 2021 11:42:33 +0200 Subject: [PATCH] extend `build_proof` for merkle multiproofs This extends the `build_proof` helpers to use the new merkle multiproof implementation from the `nim-ssz-serialization` repo. --- .gitmodules | 4 +- beacon_chain/spec/helpers.nim | 183 ++++++++++++------ .../test_fixture_merkle_single_proof.nim | 2 +- .../altair/test_fixture_sync_protocol.nim | 9 +- tests/test_helpers.nim | 17 +- vendor/nim-ssz-serialization | 2 +- 6 files changed, 140 insertions(+), 77 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6120e45846..01d70c4864 100644 --- a/.gitmodules +++ b/.gitmodules @@ -211,9 +211,9 @@ url = https://github.com/status-im/nim-taskpools [submodule "vendor/nim-ssz-serialization"] path = vendor/nim-ssz-serialization - url = https://github.com/status-im/nim-ssz-serialization.git + url = https://github.com/etan-status/nim-ssz-serialization.git ignore = untracked - branch = master + branch = merkle-multiproof [submodule "vendor/nim-websock"] path = vendor/nim-websock url = https://github.com/status-im/nim-websock.git diff --git a/beacon_chain/spec/helpers.nim b/beacon_chain/spec/helpers.nim index 407634998c..0b3193d322 100644 --- a/beacon_chain/spec/helpers.nim +++ b/beacon_chain/spec/helpers.nim @@ -88,7 +88,7 @@ iterator get_path_indices*( # https://github.com/ethereum/consensus-specs/blob/v1.1.5/ssz/merkle-proofs.md#merkle-multiproofs func get_helper_indices*( - indices: openArray[GeneralizedIndex]): seq[GeneralizedIndex] = + indices: varargs[GeneralizedIndex]): seq[GeneralizedIndex] = ## Get the generalized indices of all "extra" chunks in the tree needed ## to prove the chunks with the given generalized indices. Note that the ## decreasing order is chosen deliberately to ensure equivalence to the order @@ -109,17 +109,20 @@ func get_helper_indices*( # https://github.com/ethereum/consensus-specs/blob/v1.1.5/ssz/merkle-proofs.md#merkle-multiproofs func check_multiproof_acceptable*( - indices: openArray[GeneralizedIndex]): Result[void, string] = + indices: varargs[GeneralizedIndex]): Result[void, string] = # Check that proof verification won't allocate excessive amounts of memory. const max_multiproof_complexity = nextPowerOfTwo(256) if indices.len > max_multiproof_complexity: - trace "Max multiproof complexity exceeded", - num_indices=indices.len, max_multiproof_complexity + when nimvm: + discard + else: + trace "Max multiproof complexity exceeded", + num_indices=indices.len, max_multiproof_complexity return err("Unsupported multiproof complexity (" & $indices.len & ")") if indices.len == 0: return err("No indices specified") - if indices.anyIt(it == 0.GeneralizedIndex): + if indices.anyIt(it <= 0.GeneralizedIndex): return err("Invalid index specified") ok() @@ -246,8 +249,7 @@ func calculate_multi_merkle_root*( if leaves.len != indices.len: return err("Length mismatch for leaves and indices") ? check_multiproof_acceptable(indices) - calculate_multi_merkle_root_impl( - leaves, proof, indices, helper_indices) + calculate_multi_merkle_root_impl(leaves, proof, indices, helper_indices) func calculate_multi_merkle_root*( leaves: openArray[Eth2Digest], @@ -256,8 +258,18 @@ func calculate_multi_merkle_root*( if leaves.len != indices.len: return err("Length mismatch for leaves and indices") ? check_multiproof_acceptable(indices) - calculate_multi_merkle_root_impl( - leaves, proof, indices, get_helper_indices(indices)) + let helper_indices = get_helper_indices(indices) + calculate_multi_merkle_root_impl(leaves, proof, indices, helper_indices) + +func calculate_multi_merkle_root*( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: static openArray[GeneralizedIndex]): Result[Eth2Digest, string] = + if leaves.len != indices.len: + return err("Length mismatch for leaves and indices") + static: ? check_multiproof_acceptable(indices) + const helper_indices = get_helper_indices(indices) + calculate_multi_merkle_root_impl(leaves, proof, indices, helper_indices) # https://github.com/ethereum/consensus-specs/blob/v1.1.2/ssz/merkle-proofs.md#merkle-multiproofs func verify_merkle_multiproof*( @@ -279,6 +291,106 @@ func verify_merkle_multiproof*( if calc.isErr: return false calc.get == root +func verify_merkle_multiproof*( + leaves: openArray[Eth2Digest], + proof: openArray[Eth2Digest], + indices: static openArray[GeneralizedIndex], + root: Eth2Digest): bool = + const helper_indices = get_helper_indices(indices) + let calc = calculate_multi_merkle_root(leaves, proof, indices, helper_indices) + if calc.isErr: return false + calc.get == root + +# https://github.com/ethereum/consensus-specs/blob/v1.1.3/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21 +func build_proof*( + anchor: auto, + indices: openArray[GeneralizedIndex], + helper_indices: openArray[GeneralizedIndex], + proof: var openArray[Eth2Digest]): Result[void, string] = + doAssert proof.len == helper_indices.len + ? check_multiproof_acceptable(indices) + hash_tree_root(anchor, helper_indices, proof) + +func build_proof*( + anchor: auto, + indices: openArray[GeneralizedIndex], + proof: var openArray[Eth2Digest]): Result[void, string] = + ? check_multiproof_acceptable(indices) + let helper_indices = get_helper_indices(indices) + doAssert proof.len == helper_indices.len + hash_tree_root(anchor, helper_indices, proof) + +func build_proof*( + anchor: auto, + indices: static openArray[GeneralizedIndex], + proof: var openArray[Eth2Digest]): Result[void, string] = + const v = check_multiproof_acceptable(indices) + when v.isErr: + result.err(v.error) + else: + const helper_indices = get_helper_indices(indices) + doAssert proof.len == helper_indices.len + hash_tree_root(anchor, helper_indices, proof) + +func build_proof*( + anchor: auto, + index: GeneralizedIndex, + proof: var openArray[Eth2Digest]): Result[void, string] = + ? check_multiproof_acceptable(index) + let helper_indices = get_helper_indices(index) + doAssert proof.len == helper_indices.len + hash_tree_root(anchor, helper_indices, proof) + +func build_proof*( + anchor: auto, + index: static GeneralizedIndex, + proof: var openArray[Eth2Digest]): Result[void, string] = + const v = check_multiproof_acceptable(index) + when v.isErr: + result.err(v.error) + else: + const helper_indices = get_helper_indices(index) + doAssert proof.len == helper_indices.len + hash_tree_root(anchor, helper_indices, proof) + +func build_proof*( + anchor: auto, + indices: openArray[GeneralizedIndex] +): Result[seq[Eth2Digest], string] = + ? check_multiproof_acceptable(indices) + let helper_indices = get_helper_indices(indices) + hash_tree_root(anchor, helper_indices) + +func build_proof*( + anchor: auto, + indices: static openArray[GeneralizedIndex] +): auto = + const v = check_multiproof_acceptable(indices) + when v.isErr: + Result[array[0, Eth2Digest], string].err(v.error) + else: + const helper_indices = get_helper_indices(indices) + hash_tree_root(anchor, helper_indices) + +func build_proof*( + anchor: auto, + index: GeneralizedIndex +): Result[seq[Eth2Digest], string] = + ? check_multiproof_acceptable(index) + let helper_indices = get_helper_indices(index) + hash_tree_root(anchor, helper_indices) + +func build_proof*( + anchor: auto, + index: static GeneralizedIndex +): auto = + const v = check_multiproof_acceptable(index) + when v.isErr: + Result[array[0, Eth2Digest], string].err(v.error) + else: + const helper_indices = get_helper_indices(index) + hash_tree_root(anchor, helper_indices) + # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/phase0/beacon-chain.md#is_valid_merkle_branch func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest], depth: int, index: uint64, @@ -299,59 +411,6 @@ func is_valid_merkle_branch*(leaf: Eth2Digest, branch: openArray[Eth2Digest], value = eth2digest(buf) value == root -# https://github.com/ethereum/consensus-specs/blob/v1.1.3/tests/core/pyspec/eth2spec/test/helpers/merkle.py#L4-L21 -func build_proof_impl(anchor: object, leaf_index: uint64, - proof: var openArray[Eth2Digest]) = - let - bottom_length = nextPow2(typeof(anchor).totalSerializedFields.uint64) - tree_depth = log2trunc(bottom_length) - parent_index = - if leaf_index < bottom_length shl 1: - 0'u64 - else: - var i = leaf_index - while i >= bottom_length shl 1: - i = i shr 1 - i - - var - prefix_len = 0 - proof_len = log2trunc(leaf_index) - cache = newSeq[Eth2Digest](bottom_length shl 1) - block: - var i = bottom_length - anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar): - if i == parent_index: - when fieldVar is object: - prefix_len = log2trunc(leaf_index) - tree_depth - proof_len -= prefix_len - let - bottom_bits = leaf_index and not (uint64.high shl prefix_len) - prefix_leaf_index = (1'u64 shl prefix_len) + bottom_bits - build_proof_impl(fieldVar, prefix_leaf_index, proof) - else: raiseAssert "Invalid leaf_index" - cache[i] = hash_tree_root(fieldVar) - i += 1 - for i in countdown(bottom_length - 1, 1): - cache[i] = withEth2Hash: - h.update cache[i shl 1].data - h.update cache[i shl 1 + 1].data - - var i = if parent_index != 0: parent_index - else: leaf_index - doAssert i > 0 and i < bottom_length shl 1 - for proof_index in prefix_len ..< prefix_len + proof_len: - let b = (i and 1) != 0 - i = i shr 1 - proof[proof_index] = if b: cache[i shl 1] - else: cache[i shl 1 + 1] - -func build_proof*(anchor: object, leaf_index: uint64, - proof: var openArray[Eth2Digest]) = - doAssert leaf_index > 0 - doAssert proof.len == log2trunc(leaf_index) - build_proof_impl(anchor, leaf_index, proof) - # https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/altair/validator.md#sync-committee template sync_committee_period*(epoch: Epoch): SyncCommitteePeriod = (epoch div EPOCHS_PER_SYNC_COMMITTEE_PERIOD).SyncCommitteePeriod diff --git a/tests/consensus_spec/altair/test_fixture_merkle_single_proof.nim b/tests/consensus_spec/altair/test_fixture_merkle_single_proof.nim index 37dc78f510..00e1ee9543 100644 --- a/tests/consensus_spec/altair/test_fixture_merkle_single_proof.nim +++ b/tests/consensus_spec/altair/test_fixture_merkle_single_proof.nim @@ -47,7 +47,7 @@ proc runTest(identifier: string) = altair.BeaconState)) var computedProof = newSeq[Eth2Digest](log2trunc(proof.leaf_index)) - build_proof(state[], proof.leaf_index, computedProof) + build_proof(state[], proof.leaf_index, computedProof).get check: computedProof == proof.branch.mapIt(Eth2Digest.fromHex(it)) diff --git a/tests/consensus_spec/altair/test_fixture_sync_protocol.nim b/tests/consensus_spec/altair/test_fixture_sync_protocol.nim index affb1a4ac5..12e157186f 100644 --- a/tests/consensus_spec/altair/test_fixture_sync_protocol.nim +++ b/tests/consensus_spec/altair/test_fixture_sync_protocol.nim @@ -176,9 +176,8 @@ suite "Ethereum Foundation - Altair - Unittests - Sync protocol" & preset(): forked[], committee, block_root = block_header.hash_tree_root()) # Sync committee is updated - var next_sync_committee_branch {.noinit.}: - array[log2trunc(NEXT_SYNC_COMMITTEE_INDEX), Eth2Digest] - build_proof(state, NEXT_SYNC_COMMITTEE_INDEX, next_sync_committee_branch) + let next_sync_committee_branch = + build_proof(state, NEXT_SYNC_COMMITTEE_INDEX).get # Finality is unchanged let finality_header = BeaconBlockHeader() var finality_branch: array[log2trunc(FINALIZED_ROOT_INDEX), Eth2Digest] @@ -246,9 +245,7 @@ suite "Ethereum Foundation - Altair - Unittests - Sync protocol" & preset(): compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) finalized_block_header.hash_tree_root() == state.finalized_checkpoint.root - var finality_branch {.noinit.}: - array[log2trunc(FINALIZED_ROOT_INDEX), Eth2Digest] - build_proof(state, FINALIZED_ROOT_INDEX, finality_branch) + let finality_branch = build_proof(state, FINALIZED_ROOT_INDEX).get # Build block header let diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index 4f47a5963c..ed492cff0c 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -47,9 +47,11 @@ suite "Spec helpers": anchor.enumInstanceSerializedFields(fieldNameVar, fieldVar): let depth = log2trunc(i) var proof = newSeq[Eth2Digest](depth) - build_proof(state, i, proof) - check: is_valid_merkle_branch(hash_tree_root(fieldVar), proof, - depth, get_subtree_index(i), root) + build_proof(state, i, proof).get + check: + hash_tree_root(fieldVar) == hash_tree_root(state, i).get + is_valid_merkle_branch(hash_tree_root(fieldVar), proof, + depth, get_subtree_index(i), root) when fieldVar is object and not (fieldVar is Eth2Digest): let numChildLeaves = fieldVar.numLeaves @@ -96,9 +98,13 @@ suite "Spec helpers": ] test "verify_merkle_multiproof": + var allLeaves: array[8, Eth2Digest] + for i in 0 ..< allLeaves.len: + allLeaves[i] = eth2digest([i.byte]) + var nodes: array[16, Eth2Digest] - for i in countdown(15, 8): - nodes[i] = eth2digest([i.byte]) + for i in 0 ..< allLeaves.len: + nodes[i + 8] = allLeaves[i] for i in countdown(7, 1): nodes[i] = withEth2Hash: h.update nodes[2 * i + 0].data @@ -113,6 +119,7 @@ suite "Spec helpers": root = nodes[1] checkpoint "Verifying " & $indices & "---" & $helper_indices check: + proof == build_proof(allLeaves, indices).get verify_merkle_multiproof(leaves, proof, indices, root) verify([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) diff --git a/vendor/nim-ssz-serialization b/vendor/nim-ssz-serialization index 1cb21eda4a..efeccb0cdb 160000 --- a/vendor/nim-ssz-serialization +++ b/vendor/nim-ssz-serialization @@ -1 +1 @@ -Subproject commit 1cb21eda4ab02a6ec87839dba1beb4d4a5de127d +Subproject commit efeccb0cdb6b1f5370dcf7feafbcd3a675219e68