fix: clean up tree definitions (#1655)

There were quite a few public type definitions, that were mostly replaced by
the `SectorShape*` types. This commit cleans them up and moves them around if
appropriate.

This should make the code easier to follow and the public API surface smaller.

BREAKING CHANGE: `BinaryLCMerkleTree`, `BinaryMerkleTree`,
`BinarySubMerkleTree`, `LCMerkleTree`, `LCStore`, `MerkleStore`, `MerkleTree`,
`OctLCMerkleTree`, `OctLCSubMerkleTree`, `OctLCTopMerkleTree`, `OctMerkleTree`,
`OctSubMerkleTree`, `OctTopMerkleTree`, `QuadLCMerkleTree` and `QuadMerkleTree`
are removed from the public interface.

fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs

@@ -16,14 +16,13 @@ use generic_array::typenum::Unsigned;
 use memmap2::MmapOptions;
 use merkletree::{
     merkle::get_merkle_tree_len,
-    store::{ExternalReader, ReplicaConfig, Store, StoreConfig},
+    store::{ExternalReader, LevelCacheStore, ReplicaConfig, Store, StoreConfig},
 };
 use pasta_curves::{Fp, Fq};
 use storage_proofs_core::{
     cache_key::CacheKey,
     merkle::{
-        create_lc_tree, get_base_tree_count, split_config_and_replica, LCStore, LCTree,
-        MerkleTreeTrait,
+        create_lc_tree, get_base_tree_count, split_config_and_replica, LCTree, MerkleTreeTrait,
     },
     util::{default_rows_to_discard, NODE_SIZE},
 };
@@ -75,14 +74,22 @@ where
     let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?;
     let tree_r_last_root = if is_sector_shape_base(sector_size) {
         ensure!(configs.len() == 1, "Invalid tree-shape specified");
-        let store = LCStore::<DefaultTreeDomain<F>>::new_from_disk_with_reader(
+//<<<<<<< HEAD
+//        let store = LCStore::<DefaultTreeDomain<F>>::new_from_disk_with_reader(
+//=======
+        let store = LevelCacheStore::new_from_disk_with_reader(
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
             base_tree_len,
             OCT_ARITY,
             &configs[0],
             ExternalReader::new_from_path(&replica_config.path)?,
         )?;
+//<<<<<<< HEAD
 
-        let tree_r_last = SectorShapeBase::<F>::from_data_store(store, base_tree_leafs)?;
+//        let tree_r_last = SectorShapeBase::<F>::from_data_store(store, base_tree_leafs)?;
+//=======
+        let tree_r_last = SectorShapeBase::from_data_store(store, base_tree_leafs)?;
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
         tree_r_last.root()
     } else if is_sector_shape_sub2(sector_size) {
         let tree_r_last = SectorShapeSub2::<F>::from_store_configs_and_replica(
@@ -255,7 +262,7 @@ where
     // First, read the roots from the cached trees on disk
     let mut cached_base_tree_roots: Vec<DefaultTreeDomain<F>> = Vec::with_capacity(tree_count);
     for (i, config) in configs.iter().enumerate().take(tree_count) {
-        let store = LCStore::new_from_disk_with_reader(
+        let store = LevelCacheStore::new_from_disk_with_reader(
             base_tree_len,
             OCT_ARITY,
             config,

filecoin-proofs/src/constants.rs

@@ -7,7 +7,7 @@ pub use storage_proofs_porep::stacked::EXP_DEGREE;
 use filecoin_hashers::{poseidon::PoseidonHasher, sha256::Sha256Hasher, Hasher};
 use lazy_static::lazy_static;
 use storage_proofs_core::{
-    merkle::{BinaryMerkleTree, LCTree, OctLCMerkleTree, OctMerkleTree},
+    merkle::{BinaryMerkleTree, DiskTree, LCTree},
     util::NODE_SIZE,
     MAX_LEGACY_POREP_REGISTERED_PROOF_ID,
 };
@@ -164,11 +164,23 @@ pub type DefaultPieceDomain<F> = <DefaultPieceHasher<F> as Hasher>::Domain;
 pub type DefaultTreeHasher<F> = PoseidonHasher<F>;
 pub type DefaultTreeDomain<F> = <DefaultTreeHasher<F> as Hasher>::Domain;
 
+//<<<<<<< HEAD
 pub type DefaultOctTreeStore<F> = storage_proofs_core::merkle::LCStore<DefaultTreeDomain<F>>;
 
-pub type DefaultBinaryTree<F> = BinaryMerkleTree<DefaultPieceHasher<F>>;
-pub type DefaultOctTree<F> = OctMerkleTree<DefaultTreeHasher<F>>;
-pub type DefaultOctLCTree<F> = OctLCMerkleTree<DefaultTreeHasher<F>>;
+//pub type DefaultBinaryTree<F> = BinaryMerkleTree<DefaultPieceHasher<F>>;
+//pub type DefaultOctTree<F> = OctMerkleTree<DefaultTreeHasher<F>>;
+//pub type DefaultOctLCTree<F> = OctLCMerkleTree<DefaultTreeHasher<F>>;
+//=======
+/// A binary merkle tree with Poseidon hashing, where all levels have arity 2. It's fully
+/// persisted to disk.
+pub type DefaultBinaryTree<F> = BinaryMerkleTree<DefaultTreeHasher<F>>;
+/// A merkle tree with Poseidon hashing, where all levels have arity 8. It's fully persisted to
+/// disk.
+pub type DefaultOctTree<F> = DiskTree<DefaultTreeHasher<F>, U8, U0, U0>;
+/// A merkle tree with Poseidon hashing, where all levels have arity 8. Some levels are not
+/// persisted to disk, but only cached in memory.
+pub type DefaultOctLCTree<F> = LCTree<DefaultTreeHasher<F>, U8, U0, U0>;
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
 
 // Generic shapes
 pub type SectorShapeBase<F> = LCTree<DefaultTreeHasher<F>, U8, U0, U0>;

storage-proofs-core/src/merkle/builders.rs

@@ -19,10 +19,14 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 
 use crate::{
     error::{Error, Result},
-    merkle::{DiskTree, LCMerkleTree, LCStore, LCTree, MerkleTreeTrait, MerkleTreeWrapper},
+    merkle::{DiskTree, LCTree, MerkleTreeTrait, MerkleTreeWrapper},
     util::{data_at_node, default_rows_to_discard, NODE_SIZE},
 };
 
+/// A tree where the specified arity is used for all the levels. Some levels are not persisted to
+/// disk, but only cached in memory.
+type LCMerkleTree<H, U> = LCTree<H, U, U0, U0>;
+
 // Create a DiskTree from the provided config(s), each representing a 'base' layer tree with 'base_tree_len' elements.
 pub fn create_disk_tree<Tree: MerkleTreeTrait>(
     base_tree_len: usize,
@@ -76,7 +80,7 @@ pub fn create_lc_tree<Tree: MerkleTreeTrait>(
         LCTree::from_store_configs_and_replica(base_tree_leafs, configs, replica_config)
     } else {
         ensure!(configs.len() == 1, "Invalid tree-shape specified");
-        let store = LCStore::new_from_disk_with_reader(
+        let store = LevelCacheStore::new_from_disk_with_reader(
             base_tree_len,
             Tree::Arity::to_usize(),
             &configs[0],

storage-proofs-core/src/merkle/mod.rs

@@ -5,7 +5,7 @@ use std::fs::File;
 pub use merkletree::store::{DiskStore, ExternalReader, Store};
 
 use filecoin_hashers::Hasher;
-use generic_array::typenum::{U0, U2, U4, U8};
+use generic_array::typenum::{U0, U2};
 use merkletree::store::LevelCacheStore;
 
 mod builders;
@@ -18,26 +18,28 @@ pub use tree::*;
 
 pub type LCStore<E> = LevelCacheStore<E, File>;
 
-pub type MerkleStore<T> = DiskStore<T>;
-
+/// A tree that is fully persisted to disk.
+///
+/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top-tree
+/// arity `W`.
+///
+/// The base arity is used for for all levels up to the top. Non zero arties of the top-tree and/or
+/// sub-tree each add another layer on top. So a tree with e.g. `U = 8`, `V = 4`, `W = 2` would
+/// create a tree where the top level has two children, the levels below 4 children and all other
+/// levels below have 8 children.
 pub type DiskTree<H, U, V, W> = MerkleTreeWrapper<H, DiskStore<<H as Hasher>::Domain>, U, V, W>;
-pub type LCTree<H, U, V, W> = MerkleTreeWrapper<H, LCStore<<H as Hasher>::Domain>, U, V, W>;
-
-pub type MerkleTree<H, U> = DiskTree<H, U, U0, U0>;
-pub type LCMerkleTree<H, U> = LCTree<H, U, U0, U0>;
-
-pub type BinaryMerkleTree<H> = MerkleTree<H, U2>;
-pub type BinaryLCMerkleTree<H> = LCMerkleTree<H, U2>;
-
-pub type BinarySubMerkleTree<H> = DiskTree<H, U2, U2, U0>;
-
-pub type QuadMerkleTree<H> = MerkleTree<H, U4>;
-pub type QuadLCMerkleTree<H> = LCMerkleTree<H, U4>;
-
-pub type OctMerkleTree<H> = DiskTree<H, U8, U0, U0>;
-pub type OctSubMerkleTree<H> = DiskTree<H, U8, U2, U0>;
-pub type OctTopMerkleTree<H> = DiskTree<H, U8, U8, U2>;
 
-pub type OctLCMerkleTree<H> = LCTree<H, U8, U0, U0>;
-pub type OctLCSubMerkleTree<H> = LCTree<H, U8, U2, U0>;
-pub type OctLCTopMerkleTree<H> = LCTree<H, U8, U8, U2>;
+/// A tree that is partially stored on disk, some levels are in memory.
+///
+/// It's generic over the hash function `H`, the base arity `U`, sub-tree arity `V` and top tree
+/// arity `W`.
+///
+/// The base arity is used for for all levels up to the top. Non zero arties of the top-tree and/or
+/// sub-tree each add another layer on top. So a tree with e.g. `U = 8`, `V = 4`, `W = 2` would
+/// create a tree where the top level has two children, the levels below 4 children and all other
+/// levels below have 8 children.
+pub type LCTree<H, U, V, W> =
+    MerkleTreeWrapper<H, LevelCacheStore<<H as Hasher>::Domain, File>, U, V, W>;
+
+/// A binary merkle tree, where all levels have arity 2. It's fully persisted to disk.
+pub type BinaryMerkleTree<H> = DiskTree<H, U2, U0, U0>;

storage-proofs-porep/src/drg/vanilla.rs

@@ -4,7 +4,7 @@ use std::path::PathBuf;
 use anyhow::{ensure, Context};
 use filecoin_hashers::{Domain, HashFunction, Hasher, PoseidonArity};
 use fr32::bytes_into_fr_repr_safe;
-use generic_array::typenum::U2;
+use generic_array::typenum::{U0, U2};
 use merkletree::store::{ReplicaConfig, StoreConfig};
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 use serde::{Deserialize, Serialize};
@@ -16,8 +16,8 @@ use storage_proofs_core::{
     drgraph::Graph,
     error::Result,
     merkle::{
-        create_base_lcmerkle_tree, create_base_merkle_tree, BinaryLCMerkleTree, BinaryMerkleTree,
-        LCMerkleTree, MerkleProof, MerkleProofTrait, MerkleTreeTrait,
+        create_base_lcmerkle_tree, create_base_merkle_tree, BinaryMerkleTree, LCTree, MerkleProof,
+        MerkleProofTrait, MerkleTreeTrait,
     },
     parameter_cache::ParameterSetMetadata,
     proof::{NoRequirements, ProofScheme},
@@ -27,6 +27,10 @@ use storage_proofs_core::{
 
 use crate::{encode, PoRep};
 
+/// A binary merkle tree, where all levels have arity 2. Some levels are not persisted to disk, but
+/// only cached in memory.
+type BinaryLCMerkleTree<H> = LCTree<H, U2, U0, U0>;
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 pub struct Tau<T> {
     pub comm_r: T,
@@ -580,13 +584,22 @@ pub fn decode_domain_block<H: Hasher>(
     node: usize,
     node_data: H::Domain,
     parents: &[u32],
-) -> Result<H::Domain> {
-    let key = create_key_from_tree::<H, _>(replica_id, node, parents, tree)?;
+//<<<<<<< HEAD
+//) -> Result<H::Domain> {
+//    let key = create_key_from_tree::<H, _>(replica_id, node, parents, tree)?;
+//=======
+) -> Result<H::Domain>
+where
+    H: Hasher,
+{
+    let key = create_key_from_tree::<H>(replica_id, node, parents, tree)?;
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
 
     Ok(encode::decode(key, node_data))
 }
 
 /// Creates the encoding key from a `MerkleTree`.
+/*<<<<<<< HEAD
 /// The algorithm for that is `Blake2s(id | encodedParentNode1 | encodedParentNode1 | ...)`.
 /// It is only public so that it can be used for benchmarking
 pub fn create_key_from_tree<H, U>(
@@ -599,6 +612,15 @@ where
     H: Hasher,
     U: PoseidonArity<H::Field>,
 {
+=======*/
+/// The algorithm for that is `Poseidon(id | encodedParentNode1 | encodedParentNode1 | ...)`.
+fn create_key_from_tree<H: Hasher>(
+    id: &H::Domain,
+    node: usize,
+    parents: &[u32],
+    tree: &BinaryLCMerkleTree<H>,
+) -> Result<H::Domain> {
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
     let mut hasher = Sha256::new();
     hasher.update(AsRef::<[u8]>::as_ref(&id));
 

storage-proofs-porep/src/stacked/vanilla/proof.rs

@@ -29,7 +29,7 @@ use storage_proofs_core::{
     measurements::{measure_op, Operation},
     merkle::{
         create_disk_tree, create_lc_tree, get_base_tree_count, split_config,
-        split_config_and_replica, BinaryMerkleTree, DiskTree, LCTree, MerkleProofTrait, MerkleTree,
+        split_config_and_replica, BinaryMerkleTree, DiskTree, LCTree, MerkleProofTrait,
         MerkleTreeTrait,
     },
     settings::SETTINGS,
@@ -436,7 +436,7 @@ where
         let leafs = tree_data.len() / NODE_SIZE;
         assert_eq!(tree_data.len() % NODE_SIZE, 0);
 
-        let tree = MerkleTree::from_par_iter_with_config(
+        let tree = BinaryMerkleTree::from_par_iter_with_config(
             (0..leafs)
                 .into_par_iter()
                 // TODO: proper error handling instead of `unwrap()`

storage-proofs-post/tests/fallback_circuit.rs

@@ -12,7 +12,7 @@ use storage_proofs_core::{
     api_version::ApiVersion,
     compound_proof::CompoundProof,
     error::Result,
-    merkle::{generate_tree, get_base_tree_count, LCTree, MerkleTreeTrait, OctMerkleTree},
+    merkle::{generate_tree, get_base_tree_count, DiskTree, LCTree, MerkleTreeTrait},
     proof::ProofScheme,
     util::NODE_SIZE,
     TEST_SEED,
@@ -212,11 +212,19 @@ fn test_fallback_post_circuit_poseidon_base_8_bench_cs() {
         api_version: ApiVersion::V1_1_0,
     };
 
+/*<<<<<<< HEAD
     let pp = FallbackPoSt::<OctMerkleTree<PoseidonHasher<Fr>>>::setup(&params)
         .expect("fallback post setup failure");
 
     let mut cs = BenchCS::<Fr>::new();
     FallbackPoStCompound::<OctMerkleTree<PoseidonHasher<Fr>>>::blank_circuit(&pp)
+=======*/
+    let pp = FallbackPoSt::<DiskTree<PoseidonHasher<Fr>, U8, U0, U0>>::setup(&params)
+        .expect("fallback post setup failure");
+
+    let mut cs = BenchCS::<Fr>::new();
+    FallbackPoStCompound::<DiskTree<PoseidonHasher<Fr>, U8, U0, U0>>::blank_circuit(&pp)
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
         .synthesize(&mut cs)
         .expect("blank circuit failure");
 

storage-proofs-update/src/constants.rs

@@ -11,13 +11,20 @@ use neptune::{
     hash_type::{CType, HashType},
     poseidon::PoseidonConstants,
 };
+//<<<<<<< HEAD
 use storage_proofs_core::{
     merkle::{BinaryMerkleTree, LCTree},
     SECTOR_NODES_16_KIB, SECTOR_NODES_16_MIB, SECTOR_NODES_1_KIB, SECTOR_NODES_2_KIB,
     SECTOR_NODES_32_GIB, SECTOR_NODES_32_KIB, SECTOR_NODES_4_KIB, SECTOR_NODES_512_MIB,
     SECTOR_NODES_64_GIB, SECTOR_NODES_8_KIB, SECTOR_NODES_8_MIB,
 };
 use typemap::ShareMap;
+//=======
+//use storage_proofs_core::merkle::{BinaryMerkleTree, LCTree, MerkleTreeTrait};
+
+// Use a custom domain separation tag when generating randomness phi, rho, and challenges bits.
+pub const HASH_TYPE_GEN_RANDOMNESS: HashType<Fr, U2> = HashType::Custom(CType::Arbitrary(1));
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
 
 lazy_static! {
     // Use a custom domain separation tag `HashType` when using Poseidon to generate randomness
@@ -73,11 +80,18 @@ pub type TreeD<F> = BinaryMerkleTree<TreeDHasher<F>>;
 pub type TreeDStore<F> = DiskStore<TreeDDomain<F>>;
 pub type TreeDArity = U2;
 
+//<<<<<<< HEAD
 pub type TreeRHasher<F> = PoseidonHasher<F>;
 pub type TreeRDomain<F> = <TreeRHasher<F> as Hasher>::Domain;
 pub type TreeR<F, U, V, W> = LCTree<TreeRHasher<F>, U, V, W>;
 // All valid TreeR shapes have the same base-tree shape.
 pub type TreeRBase<F> = LCTree<TreeRHasher<F>, U8, U0, U0>;
+//=======
+//pub type TreeRHasher = PoseidonHasher;
+//pub type TreeRDomain = PoseidonDomain;
+// All valid TreeR's have the same base-tree shape.
+//pub type TreeRBaseTree = LCTree<TreeRHasher, U8, U0, U0>;
+//>>>>>>> 128f7209 (fix: clean up tree definitions (#1655))
 
 // The number of groth16 partitions for the given sector size.
 pub const fn partition_count(sector_nodes: usize) -> usize {