diff --git a/filecoin-proofs/src/api/seal.rs b/filecoin-proofs/src/api/seal.rs index f34e37d75..9b9efecb2 100644 --- a/filecoin-proofs/src/api/seal.rs +++ b/filecoin-proofs/src/api/seal.rs @@ -758,3 +758,52 @@ pub fn verify_batch_seal( info!("verify_batch_seal:finish"); result } + +pub fn fauxrep, S: AsRef, Tree: 'static + MerkleTreeTrait>( + porep_config: PoRepConfig, + cache_path: R, + out_path: S, +) -> Result { + let mut rng = rand::thread_rng(); + fauxrep_aux::<_, R, S, Tree>(&mut rng, porep_config, cache_path, out_path) +} + +pub fn fauxrep_aux< + Rng: rand::Rng, + R: AsRef, + S: AsRef, + Tree: 'static + MerkleTreeTrait, +>( + mut rng: &mut Rng, + porep_config: PoRepConfig, + cache_path: R, + out_path: S, +) -> Result { + let sector_bytes = PaddedBytesAmount::from(porep_config).0; + + { + // Create a sector full of null bytes at `out_path`. + let file = File::create(&out_path)?; + file.set_len(sector_bytes)?; + } + + let fake_comm_c = ::Domain::random(&mut rng); + let (comm_r, p_aux) = StackedDrg::::fake_replicate_phase2( + fake_comm_c, + out_path, + &cache_path, + sector_bytes as usize, + )?; + + let p_aux_path = cache_path.as_ref().join(CacheKey::PAux.to_string()); + let mut f_p_aux = File::create(&p_aux_path) + .with_context(|| format!("could not create file p_aux={:?}", p_aux_path))?; + let p_aux_bytes = serialize(&p_aux)?; + f_p_aux + .write_all(&p_aux_bytes) + .with_context(|| format!("could not write to file p_aux={:?}", p_aux_path))?; + + let mut commitment = [0u8; 32]; + commitment[..].copy_from_slice(&comm_r.into_bytes()[..]); + Ok(commitment) +} diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index 2573fe396..95b06efd5 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -81,25 +81,29 @@ fn seal_lifecycle(sector_size: u64) -> Result<( #[test] #[ignore] fn test_winning_post_2kib_base_8() -> Result<()> { - winning_post::(SECTOR_SIZE_2_KIB) + winning_post::(SECTOR_SIZE_2_KIB, false)?; + winning_post::(SECTOR_SIZE_2_KIB, true) } #[test] #[ignore] fn test_winning_post_4kib_sub_8_2() -> Result<()> { - winning_post::(SECTOR_SIZE_4_KIB) + winning_post::(SECTOR_SIZE_4_KIB, false)?; + winning_post::(SECTOR_SIZE_4_KIB, true) } #[test] #[ignore] fn test_winning_post_16kib_sub_8_8() -> Result<()> { - winning_post::(SECTOR_SIZE_16_KIB) + winning_post::(SECTOR_SIZE_16_KIB, false)?; + winning_post::(SECTOR_SIZE_16_KIB, true) } #[test] #[ignore] fn test_winning_post_32kib_top_8_8_2() -> Result<()> { - winning_post::(SECTOR_SIZE_32_KIB) + winning_post::(SECTOR_SIZE_32_KIB, false)?; + winning_post::(SECTOR_SIZE_32_KIB, true) } #[test] @@ -138,15 +142,18 @@ fn test_winning_post_empty_sector_challenge() -> Result<()> { Ok(()) } -fn winning_post(sector_size: u64) -> Result<()> { +fn winning_post(sector_size: u64, fake: bool) -> Result<()> { let rng = &mut XorShiftRng::from_seed(TEST_SEED); let prover_fr: DefaultTreeDomain = Fr::random(rng).into(); let mut prover_id = [0u8; 32]; prover_id.copy_from_slice(AsRef::<[u8]>::as_ref(&prover_fr)); - let (sector_id, replica, comm_r, cache_dir) = - create_seal::<_, Tree>(rng, sector_size, prover_id, true)?; + let (sector_id, replica, comm_r, cache_dir) = if fake { + create_fake_seal::<_, Tree>(rng, sector_size)? + } else { + create_seal::<_, Tree>(rng, sector_size, prover_id, true)? + }; let sector_count = WINNING_POST_SECTOR_COUNT; let random_fr: DefaultTreeDomain = Fr::random(rng).into(); @@ -194,7 +201,8 @@ fn test_window_post_single_partition_smaller_2kib_base_8() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, sector_count / 2, sector_count) + window_post::(sector_size, sector_count / 2, sector_count, false)?; + window_post::(sector_size, sector_count / 2, sector_count, true) } #[test] @@ -207,7 +215,8 @@ fn test_window_post_two_partitions_matching_2kib_base_8() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, 2 * sector_count, sector_count) + window_post::(sector_size, 2 * sector_count, sector_count, false)?; + window_post::(sector_size, 2 * sector_count, sector_count, true) } #[test] @@ -220,7 +229,8 @@ fn test_window_post_two_partitions_matching_4kib_sub_8_2() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, 2 * sector_count, sector_count) + window_post::(sector_size, 2 * sector_count, sector_count, false)?; + window_post::(sector_size, 2 * sector_count, sector_count, true) } #[test] @@ -233,7 +243,8 @@ fn test_window_post_two_partitions_matching_16kib_sub_8_8() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, 2 * sector_count, sector_count) + window_post::(sector_size, 2 * sector_count, sector_count, false)?; + window_post::(sector_size, 2 * sector_count, sector_count, true) } #[test] @@ -246,7 +257,8 @@ fn test_window_post_two_partitions_matching_32kib_top_8_8_2() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, 2 * sector_count, sector_count) + window_post::(sector_size, 2 * sector_count, sector_count, false)?; + window_post::(sector_size, 2 * sector_count, sector_count, true) } #[test] @@ -259,7 +271,8 @@ fn test_window_post_two_partitions_smaller_2kib_base_8() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, 2 * sector_count - 1, sector_count) + window_post::(sector_size, 2 * sector_count - 1, sector_count, false)?; + window_post::(sector_size, 2 * sector_count - 1, sector_count, true) } #[test] @@ -272,13 +285,15 @@ fn test_window_post_single_partition_matching_2kib_base_8() -> Result<()> { .get(§or_size) .unwrap(); - window_post::(sector_size, sector_count, sector_count) + window_post::(sector_size, sector_count, sector_count, false)?; + window_post::(sector_size, sector_count, sector_count, true) } fn window_post( sector_size: u64, total_sector_count: usize, sector_count: usize, + fake: bool, ) -> Result<()> { let rng = &mut XorShiftRng::from_seed(TEST_SEED); @@ -291,8 +306,11 @@ fn window_post( prover_id.copy_from_slice(AsRef::<[u8]>::as_ref(&prover_fr)); for _ in 0..total_sector_count { - let (sector_id, replica, comm_r, cache_dir) = - create_seal::<_, Tree>(rng, sector_size, prover_id, true)?; + let (sector_id, replica, comm_r, cache_dir) = if fake { + create_fake_seal::<_, Tree>(rng, sector_size)? + } else { + create_seal::<_, Tree>(rng, sector_size, prover_id, true)? + }; priv_replicas.insert( sector_id, PrivateReplicaInfo::new(replica.path().into(), comm_r, cache_dir.path().into())?, @@ -464,3 +482,34 @@ fn create_seal( Ok((sector_id, sealed_sector_file, comm_r, cache_dir)) } + +fn create_fake_seal( + mut rng: &mut R, + sector_size: u64, +) -> Result<(SectorId, NamedTempFile, Commitment, tempfile::TempDir)> { + init_logger(); + + let arbitrary_porep_id = [28; 32]; + let sealed_sector_file = NamedTempFile::new()?; + + let config = PoRepConfig { + sector_size: SectorSize(sector_size), + partitions: PoRepProofPartitions( + *POREP_PARTITIONS.read().unwrap().get(§or_size).unwrap(), + ), + porep_id: arbitrary_porep_id, + }; + + let cache_dir = tempfile::tempdir().unwrap(); + + let sector_id = rng.gen::().into(); + + let comm_r = fauxrep_aux::<_, _, _, Tree>( + &mut rng, + config, + cache_dir.path(), + sealed_sector_file.path(), + )?; + + Ok((sector_id, sealed_sector_file, comm_r, cache_dir)) +} diff --git a/storage-proofs/porep/src/stacked/vanilla/proof.rs b/storage-proofs/porep/src/stacked/vanilla/proof.rs index b316b97c1..fe7ac93dc 100644 --- a/storage-proofs/porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/proof.rs @@ -1,7 +1,7 @@ use std::fs::OpenOptions; use std::io::Write; use std::marker::PhantomData; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::{mpsc, Arc, RwLock}; use generic_array::typenum::{self, Unsigned}; @@ -1161,6 +1161,168 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr Ok((tau, (paux, taux))) } + + // Assumes data is all zeros. + // Replica path is used to create configs, but is not read. + // Instead new zeros are provided (hence the need for replica to be all zeros). + fn generate_fake_tree_r_last( + nodes_count: usize, + tree_count: usize, + tree_r_last_config: StoreConfig, + replica_path: PathBuf, + ) -> Result> + where + TreeArity: PoseidonArity, + { + let (configs, replica_config) = split_config_and_replica( + tree_r_last_config.clone(), + replica_path, + nodes_count, + tree_count, + )?; + + if settings::SETTINGS.lock().unwrap().use_gpu_tree_builder { + info!("generating tree r last using the GPU"); + let max_gpu_tree_batch_size = + settings::SETTINGS.lock().unwrap().max_gpu_tree_batch_size as usize; + + let mut tree_builder = TreeBuilder::::new( + Some(BatcherType::GPU), + nodes_count, + max_gpu_tree_batch_size, + tree_r_last_config.rows_to_discard, + ) + .expect("failed to create TreeBuilder"); + + // Allocate zeros once and reuse. + let zero_leaves: Vec = vec![Fr::zero(); max_gpu_tree_batch_size]; + for (i, config) in configs.iter().enumerate() { + let mut consumed = 0; + while consumed < nodes_count { + let batch_size = usize::min(max_gpu_tree_batch_size, nodes_count - consumed); + + consumed += batch_size; + + if consumed != nodes_count { + tree_builder + .add_leaves(&zero_leaves[0..batch_size]) + .expect("failed to add leaves"); + continue; + }; + + // If we get here, this is a final leaf batch: build a sub-tree. + info!( + "building base tree_r_last with GPU {}/{}", + i + 1, + tree_count + ); + + let (_, tree_data) = tree_builder + .add_final_leaves(&zero_leaves[0..batch_size]) + .expect("failed to add final leaves"); + let tree_data_len = tree_data.len(); + let cache_size = get_merkle_tree_cache_size( + get_merkle_tree_leafs(config.size.unwrap(), Tree::Arity::to_usize()) + .expect("failed to get merkle tree leaves"), + Tree::Arity::to_usize(), + config.rows_to_discard, + ) + .expect("failed to get merkle tree cache size"); + assert_eq!(tree_data_len, cache_size); + + let flat_tree_data: Vec<_> = tree_data + .into_par_iter() + .flat_map(|el| fr_into_bytes(&el)) + .collect(); + + // Persist the data to the store based on the current config. + let tree_r_last_path = StoreConfig::data_path(&config.path, &config.id); + trace!( + "persisting tree r of len {} with {} rows to discard at path {:?}", + tree_data_len, + config.rows_to_discard, + tree_r_last_path + ); + let mut f = OpenOptions::new() + .create(true) + .write(true) + .open(&tree_r_last_path) + .expect("failed to open file for tree_r_last"); + f.write_all(&flat_tree_data) + .expect("failed to wrote tree_r_last data"); + } + } + } else { + info!("generating tree r last using the CPU"); + for (i, config) in configs.iter().enumerate() { + let encoded_data = vec![::Domain::default(); nodes_count]; + + info!( + "building base tree_r_last with CPU {}/{}", + i + 1, + tree_count + ); + LCTree::::from_par_iter_with_config(encoded_data, config.clone())?; + } + }; + + create_lc_tree::>( + tree_r_last_config.size.unwrap(), + &configs, + &replica_config, + ) + } + + pub fn fake_replicate_phase2, S: AsRef>( + tree_c_root: ::Domain, + replica_path: R, + cache_path: S, + sector_size: usize, + ) -> Result<( + ::Domain, + PersistentAux<::Domain>, + )> { + let leaf_count = sector_size / NODE_SIZE; + let replica_pathbuf = PathBuf::from(replica_path.as_ref()); + assert_eq!(0, sector_size % NODE_SIZE); + let tree_count = get_base_tree_count::(); + let nodes_count = leaf_count / tree_count; + + let config = StoreConfig::new( + cache_path.as_ref(), + CacheKey::CommRLastTree.to_string(), + default_rows_to_discard(nodes_count, Tree::Arity::to_usize()), + ); + let tree_r_last_config = StoreConfig::from_config( + &config, + CacheKey::CommRLastTree.to_string(), + Some(get_merkle_tree_len(nodes_count, Tree::Arity::to_usize())?), + ); + + // Encode original data into the last layer. + info!("building tree_r_last"); + let tree_r_last = Self::generate_fake_tree_r_last::( + nodes_count, + tree_count, + tree_r_last_config, + replica_pathbuf, + )?; + info!("tree_r_last done"); + + let tree_r_last_root = tree_r_last.root(); + drop(tree_r_last); + + // comm_r = H(comm_c || comm_r_last) + let comm_r: ::Domain = + ::Function::hash2(&tree_c_root, &tree_r_last_root); + + let p_aux = PersistentAux { + comm_c: tree_c_root, + comm_r_last: tree_r_last_root, + }; + + Ok((comm_r, p_aux)) + } } #[cfg(test)]