From dff46ea55c5a01864b0eb604d3251473b7e079f9 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Tue, 20 Dec 2022 12:00:24 +0100 Subject: [PATCH 1/4] fix: clean up tree definitions There were quite a few public type definitions, that were mostly replaced by the `SectorShape*` types. This commit cleans them up and moves them around if appropriate. This should make the code easier to follow and the public API surface smaller. BREAKING CHANGE: `BinaryLCMerkleTree`, `BinaryMerkleTree`, `BinarySubMerkleTree`, `LCMerkleTree`, `LCStore`, `MerkleStore`, `MerkleTree`, `OctLCMerkleTree`, `OctLCSubMerkleTree`, `OctLCTopMerkleTree`, `OctMerkleTree`, `OctSubMerkleTree`, `OctTopMerkleTree`, `QuadLCMerkleTree` and `QuadMerkleTree` are removed from the public interface. --- .../src/bin/update_tree_r_cache/main.rs | 10 ++-- filecoin-proofs/src/constants.rs | 12 +++-- storage-proofs-core/src/merkle/builders.rs | 8 ++- storage-proofs-core/src/merkle/mod.rs | 50 ++++++++++--------- storage-proofs-porep/src/drg/vanilla.rs | 19 ++++--- .../src/stacked/vanilla/proof.rs | 4 +- storage-proofs-post/tests/fallback_circuit.rs | 6 +-- storage-proofs-update/src/constants.rs | 3 +- 8 files changed, 62 insertions(+), 50 deletions(-) diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index 83dd551e4..7a8389491 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -14,13 +14,12 @@ use generic_array::typenum::Unsigned; use memmap2::MmapOptions; use merkletree::{ merkle::get_merkle_tree_len, - store::{ExternalReader, ReplicaConfig, Store, StoreConfig}, + store::{ExternalReader, LevelCacheStore, ReplicaConfig, Store, StoreConfig}, }; use storage_proofs_core::{ cache_key::CacheKey, merkle::{ - create_lc_tree, get_base_tree_count, split_config_and_replica, LCStore, LCTree, - MerkleTreeTrait, + create_lc_tree, get_base_tree_count, split_config_and_replica, LCTree, MerkleTreeTrait, }, util::{default_rows_to_discard, NODE_SIZE}, }; @@ -68,13 +67,12 @@ fn get_tree_r_last_root( let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; let tree_r_last_root = if is_sector_shape_base(sector_size) { ensure!(configs.len() == 1, "Invalid tree-shape specified"); - let store = LCStore::::new_from_disk_with_reader( + let store = LevelCacheStore::new_from_disk_with_reader( base_tree_len, OCT_ARITY, &configs[0], ExternalReader::new_from_path(&replica_config.path)?, )?; - let tree_r_last = SectorShapeBase::from_data_store(store, base_tree_leafs)?; tree_r_last.root() } else if is_sector_shape_sub2(sector_size) { @@ -223,7 +221,7 @@ fn run_verify(sector_size: usize, cache: &Path, replica_path: &Path) -> Result<( // First, read the roots from the cached trees on disk let mut cached_base_tree_roots: Vec = Vec::with_capacity(tree_count); for (i, config) in configs.iter().enumerate().take(tree_count) { - let store = LCStore::new_from_disk_with_reader( + let store = LevelCacheStore::new_from_disk_with_reader( base_tree_len, OCT_ARITY, config, diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index 064691bdc..31e5b64ab 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -7,7 +7,7 @@ pub use storage_proofs_porep::stacked::EXP_DEGREE; use filecoin_hashers::{poseidon::PoseidonHasher, sha256::Sha256Hasher, Hasher}; use lazy_static::lazy_static; use storage_proofs_core::{ - merkle::{BinaryMerkleTree, LCTree, OctLCMerkleTree, OctMerkleTree}, + merkle::{BinaryMerkleTree, DiskTree, LCTree}, util::NODE_SIZE, MAX_LEGACY_POREP_REGISTERED_PROOF_ID, }; @@ -164,9 +164,15 @@ pub type DefaultPieceDomain = ::Domain; pub type DefaultTreeHasher = PoseidonHasher; pub type DefaultTreeDomain = ::Domain; +/// A binary merkle tree with Poseidon hashing, where all levels have arity 2. It's fully +/// persisted to disk. pub type DefaultBinaryTree = BinaryMerkleTree; -pub type DefaultOctTree = OctMerkleTree; -pub type DefaultOctLCTree = OctLCMerkleTree; +/// A merkle tree with Poseidon hashing, where all levels have arity 8. It's fully persisted to +/// disk. +pub type DefaultOctTree = DiskTree; +/// A merkle tree with Poseidon hashing, where all levels have arity 8. Some levels are not +/// persisted to disk, but only cached in memory. +pub type DefaultOctLCTree = LCTree; // Generic shapes pub type SectorShapeBase = LCTree; diff --git a/storage-proofs-core/src/merkle/builders.rs b/storage-proofs-core/src/merkle/builders.rs index 28b731164..5506e777a 100644 --- a/storage-proofs-core/src/merkle/builders.rs +++ b/storage-proofs-core/src/merkle/builders.rs @@ -19,10 +19,14 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use crate::{ error::{Error, Result}, - merkle::{DiskTree, LCMerkleTree, LCStore, LCTree, MerkleTreeTrait, MerkleTreeWrapper}, + merkle::{DiskTree, LCTree, MerkleTreeTrait, MerkleTreeWrapper}, util::{data_at_node, default_rows_to_discard, NODE_SIZE}, }; +/// A tree where the specified arity is used for all the levels. Some levels are not persisted to +/// disk, but only cached in memory. +type LCMerkleTree = LCTree; + // Create a DiskTree from the provided config(s), each representing a 'base' layer tree with 'base_tree_len' elements. pub fn create_disk_tree( base_tree_len: usize, @@ -76,7 +80,7 @@ pub fn create_lc_tree( LCTree::from_store_configs_and_replica(base_tree_leafs, configs, replica_config) } else { ensure!(configs.len() == 1, "Invalid tree-shape specified"); - let store = LCStore::new_from_disk_with_reader( + let store = LevelCacheStore::new_from_disk_with_reader( base_tree_len, Tree::Arity::to_usize(), &configs[0], diff --git a/storage-proofs-core/src/merkle/mod.rs b/storage-proofs-core/src/merkle/mod.rs index ca196849c..e9ca839e0 100644 --- a/storage-proofs-core/src/merkle/mod.rs +++ b/storage-proofs-core/src/merkle/mod.rs @@ -5,7 +5,7 @@ use std::fs::File; pub use merkletree::store::{DiskStore, ExternalReader, Store}; use filecoin_hashers::Hasher; -use generic_array::typenum::{U0, U2, U4, U8}; +use generic_array::typenum::{U0, U2}; use merkletree::store::LevelCacheStore; mod builders; @@ -16,28 +16,30 @@ pub use builders::*; pub use proof::*; pub use tree::*; -pub type LCStore = LevelCacheStore; - -pub type MerkleStore = DiskStore; - +/// A tree that is fully persisted to disk. +/// +/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top tree +/// arity `W`. +/// +/// The base arity is used for the leaves. If all other arities are zero, then this arity is used +/// for all levels. +/// The sub-tree arity is for all levels between the level above the leaves and the level below the +/// root (which is always a single item). +/// The top-tree arity is used for the top level that then results in the roor node. pub type DiskTree = MerkleTreeWrapper::Domain>, U, V, W>; -pub type LCTree = MerkleTreeWrapper::Domain>, U, V, W>; - -pub type MerkleTree = DiskTree; -pub type LCMerkleTree = LCTree; - -pub type BinaryMerkleTree = MerkleTree; -pub type BinaryLCMerkleTree = LCMerkleTree; - -pub type BinarySubMerkleTree = DiskTree; - -pub type QuadMerkleTree = MerkleTree; -pub type QuadLCMerkleTree = LCMerkleTree; - -pub type OctMerkleTree = DiskTree; -pub type OctSubMerkleTree = DiskTree; -pub type OctTopMerkleTree = DiskTree; -pub type OctLCMerkleTree = LCTree; -pub type OctLCSubMerkleTree = LCTree; -pub type OctLCTopMerkleTree = LCTree; +/// A tree that is partially stored on disk, some levels are in memory. +/// +/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top tree +/// arity `W`. +/// +/// The base arity is used for the leaves. If all other arities are zero, then this arity is used +/// for all levels. +/// The sub-tree arity is for all levels between the level above the leaves and the level below the +/// root (which is always a single item). +/// The top-tree arity is used for the top level that then results in the roor node. +pub type LCTree = + MerkleTreeWrapper::Domain, File>, U, V, W>; + +/// A binary merkle tree, where all levels have arity 2. It's fully persisted to disk. +pub type BinaryMerkleTree = DiskTree; diff --git a/storage-proofs-porep/src/drg/vanilla.rs b/storage-proofs-porep/src/drg/vanilla.rs index 5793de373..ce32c695f 100644 --- a/storage-proofs-porep/src/drg/vanilla.rs +++ b/storage-proofs-porep/src/drg/vanilla.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use anyhow::{ensure, Context}; use filecoin_hashers::{Domain, HashFunction, Hasher, PoseidonArity}; use fr32::bytes_into_fr_repr_safe; -use generic_array::typenum::U2; +use generic_array::typenum::{U0, U2}; use merkletree::store::{ReplicaConfig, StoreConfig}; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; @@ -16,8 +16,8 @@ use storage_proofs_core::{ drgraph::Graph, error::Result, merkle::{ - create_base_lcmerkle_tree, create_base_merkle_tree, BinaryLCMerkleTree, BinaryMerkleTree, - LCMerkleTree, MerkleProof, MerkleProofTrait, MerkleTreeTrait, + create_base_lcmerkle_tree, create_base_merkle_tree, BinaryMerkleTree, LCTree, MerkleProof, + MerkleProofTrait, MerkleTreeTrait, }, parameter_cache::ParameterSetMetadata, proof::{NoRequirements, ProofScheme}, @@ -27,6 +27,10 @@ use storage_proofs_core::{ use crate::{encode, PoRep}; +/// A binary merkle tree, where all levels have arity 2. Some levels are not persisted to disk, but +/// only cached in memory. +type BinaryLCMerkleTree = LCTree; + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct Tau { pub comm_r: T, @@ -576,19 +580,18 @@ pub fn decode_domain_block( where H: Hasher, { - let key = create_key_from_tree::(replica_id, node, parents, tree)?; + let key = create_key_from_tree::(replica_id, node, parents, tree)?; Ok(encode::decode(key, node_data)) } /// Creates the encoding key from a `MerkleTree`. -/// The algorithm for that is `Blake2s(id | encodedParentNode1 | encodedParentNode1 | ...)`. -/// It is only public so that it can be used for benchmarking -pub fn create_key_from_tree( +/// The algorithm for that is `Poseidon(id | encodedParentNode1 | encodedParentNode1 | ...)`. +fn create_key_from_tree( id: &H::Domain, node: usize, parents: &[u32], - tree: &LCMerkleTree, + tree: &BinaryLCMerkleTree, ) -> Result { let mut hasher = Sha256::new(); hasher.update(AsRef::<[u8]>::as_ref(&id)); diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index dd64fa085..51d913f8e 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -29,7 +29,7 @@ use storage_proofs_core::{ measurements::{measure_op, Operation}, merkle::{ create_disk_tree, create_lc_tree, get_base_tree_count, split_config, - split_config_and_replica, BinaryMerkleTree, DiskTree, LCTree, MerkleProofTrait, MerkleTree, + split_config_and_replica, BinaryMerkleTree, DiskTree, LCTree, MerkleProofTrait, MerkleTreeTrait, }, settings::SETTINGS, @@ -427,7 +427,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr let leafs = tree_data.len() / NODE_SIZE; assert_eq!(tree_data.len() % NODE_SIZE, 0); - let tree = MerkleTree::from_par_iter_with_config( + let tree = BinaryMerkleTree::from_par_iter_with_config( (0..leafs) .into_par_iter() // TODO: proper error handling instead of `unwrap()` diff --git a/storage-proofs-post/tests/fallback_circuit.rs b/storage-proofs-post/tests/fallback_circuit.rs index c0727be86..04053ea6a 100644 --- a/storage-proofs-post/tests/fallback_circuit.rs +++ b/storage-proofs-post/tests/fallback_circuit.rs @@ -12,7 +12,7 @@ use storage_proofs_core::{ api_version::ApiVersion, compound_proof::CompoundProof, error::Result, - merkle::{generate_tree, get_base_tree_count, LCTree, MerkleTreeTrait, OctMerkleTree}, + merkle::{generate_tree, get_base_tree_count, DiskTree, LCTree, MerkleTreeTrait}, proof::ProofScheme, util::NODE_SIZE, TEST_SEED, @@ -211,11 +211,11 @@ fn test_fallback_post_circuit_poseidon_base_8_bench_cs() { api_version: ApiVersion::V1_1_0, }; - let pp = FallbackPoSt::>::setup(¶ms) + let pp = FallbackPoSt::>::setup(¶ms) .expect("fallback post setup failure"); let mut cs = BenchCS::::new(); - FallbackPoStCompound::>::blank_circuit(&pp) + FallbackPoStCompound::>::blank_circuit(&pp) .synthesize(&mut cs) .expect("blank circuit failure"); diff --git a/storage-proofs-update/src/constants.rs b/storage-proofs-update/src/constants.rs index 7c45aeb90..aab12a9f1 100644 --- a/storage-proofs-update/src/constants.rs +++ b/storage-proofs-update/src/constants.rs @@ -11,7 +11,7 @@ use neptune::{ poseidon::PoseidonConstants, Strength, }; -use storage_proofs_core::merkle::{BinaryMerkleTree, LCStore, LCTree, MerkleTreeTrait}; +use storage_proofs_core::merkle::{BinaryMerkleTree, LCTree, MerkleTreeTrait}; // Use a custom domain separation tag when generating randomness phi, rho, and challenges bits. pub const HASH_TYPE_GEN_RANDOMNESS: HashType = HashType::Custom(CType::Arbitrary(1)); @@ -58,7 +58,6 @@ pub type TreeDArity = U2; pub type TreeRHasher = PoseidonHasher; pub type TreeRDomain = PoseidonDomain; -pub type TreeRStore = LCStore; // All valid TreeR's have the same base-tree shape. pub type TreeRBaseTree = LCTree; From 34ee21328fa411feecd85ad0e724c682bf92b5bb Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 22 Dec 2022 13:16:26 +0100 Subject: [PATCH 2/4] Update storage-proofs-core/src/merkle/mod.rs Co-authored-by: Artem Storozhuk --- storage-proofs-core/src/merkle/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage-proofs-core/src/merkle/mod.rs b/storage-proofs-core/src/merkle/mod.rs index e9ca839e0..63f545c28 100644 --- a/storage-proofs-core/src/merkle/mod.rs +++ b/storage-proofs-core/src/merkle/mod.rs @@ -25,7 +25,7 @@ pub use tree::*; /// for all levels. /// The sub-tree arity is for all levels between the level above the leaves and the level below the /// root (which is always a single item). -/// The top-tree arity is used for the top level that then results in the roor node. +/// The top-tree arity is used for the top level that then results in the root node. pub type DiskTree = MerkleTreeWrapper::Domain>, U, V, W>; /// A tree that is partially stored on disk, some levels are in memory. From 65d90fe518121bf2947775b8d25c92a97bed9fa2 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 22 Dec 2022 13:16:33 +0100 Subject: [PATCH 3/4] Update storage-proofs-core/src/merkle/mod.rs Co-authored-by: Artem Storozhuk --- storage-proofs-core/src/merkle/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage-proofs-core/src/merkle/mod.rs b/storage-proofs-core/src/merkle/mod.rs index 63f545c28..66b4105ba 100644 --- a/storage-proofs-core/src/merkle/mod.rs +++ b/storage-proofs-core/src/merkle/mod.rs @@ -37,7 +37,7 @@ pub type DiskTree = MerkleTreeWrapper::Do /// for all levels. /// The sub-tree arity is for all levels between the level above the leaves and the level below the /// root (which is always a single item). -/// The top-tree arity is used for the top level that then results in the roor node. +/// The top-tree arity is used for the top level that then results in the root node. pub type LCTree = MerkleTreeWrapper::Domain, File>, U, V, W>; From 646f162bafdfcae74ab57d0406d298b57eb75d40 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 22 Dec 2022 14:51:33 +0100 Subject: [PATCH 4/4] docs: fix wrong comments --- storage-proofs-core/src/merkle/mod.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/storage-proofs-core/src/merkle/mod.rs b/storage-proofs-core/src/merkle/mod.rs index 66b4105ba..0b9a46199 100644 --- a/storage-proofs-core/src/merkle/mod.rs +++ b/storage-proofs-core/src/merkle/mod.rs @@ -18,14 +18,13 @@ pub use tree::*; /// A tree that is fully persisted to disk. /// -/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top tree +/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top-tree /// arity `W`. /// -/// The base arity is used for the leaves. If all other arities are zero, then this arity is used -/// for all levels. -/// The sub-tree arity is for all levels between the level above the leaves and the level below the -/// root (which is always a single item). -/// The top-tree arity is used for the top level that then results in the root node. +/// The base arity is used for for all levels up to the top. Non zero arties of the top-tree and/or +/// sub-tree each add another layer on top. So a tree with e.g. `U = 8`, `V = 4`, `W = 2` would +/// create a tree where the top level has two children, the levels below 4 children and all other +/// levels below have 8 children. pub type DiskTree = MerkleTreeWrapper::Domain>, U, V, W>; /// A tree that is partially stored on disk, some levels are in memory. @@ -33,11 +32,10 @@ pub type DiskTree = MerkleTreeWrapper::Do /// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top tree /// arity `W`. /// -/// The base arity is used for the leaves. If all other arities are zero, then this arity is used -/// for all levels. -/// The sub-tree arity is for all levels between the level above the leaves and the level below the -/// root (which is always a single item). -/// The top-tree arity is used for the top level that then results in the root node. +/// The base arity is used for for all levels up to the top. Non zero arties of the top-tree and/or +/// sub-tree each add another layer on top. So a tree with e.g. `U = 8`, `V = 4`, `W = 2` would +/// create a tree where the top level has two children, the levels below 4 children and all other +/// levels below have 8 children. pub type LCTree = MerkleTreeWrapper::Domain, File>, U, V, W>;