diff --git a/fil-proofs-tooling/src/bin/benchy/main.rs b/fil-proofs-tooling/src/bin/benchy/main.rs index 989cee72d..67c5c85c5 100644 --- a/fil-proofs-tooling/src/bin/benchy/main.rs +++ b/fil-proofs-tooling/src/bin/benchy/main.rs @@ -62,6 +62,13 @@ fn main() -> Result<()> { .help("Skip commit phase 2") .takes_value(false), ) + .arg( + Arg::with_name("test-resume") + .long("test-resume") + .required(false) + .help("Test replication resume") + .takes_value(false), + ) .arg( Arg::with_name("cache") .long("cache") @@ -168,6 +175,7 @@ fn main() -> Result<()> { let skip_precommit_phase2 = m.is_present("skip-precommit"); let skip_commit_phase1 = m.is_present("skip-commit-phase1"); let skip_commit_phase2 = m.is_present("skip-commit-phase2"); + let test_resume = m.is_present("test-resume"); let cache_dir = value_t!(m, "cache", String)?; let sector_size_kibs = value_t!(m, "size", usize) .expect("could not convert `size` CLI argument to `usize`"); @@ -180,6 +188,7 @@ fn main() -> Result<()> { skip_precommit_phase2, skip_commit_phase1, skip_commit_phase2, + test_resume, )?; } ("winning-post", Some(m)) => { diff --git a/fil-proofs-tooling/src/bin/benchy/window_post.rs b/fil-proofs-tooling/src/bin/benchy/window_post.rs index 3306acbcc..00e93e22f 100644 --- a/fil-proofs-tooling/src/bin/benchy/window_post.rs +++ b/fil-proofs-tooling/src/bin/benchy/window_post.rs @@ -102,6 +102,8 @@ fn run_pre_commit_phases( cache_dir: PathBuf, skip_precommit_phase1: bool, skip_precommit_phase2: bool, + test_resume: bool, + skip_staging: bool, ) -> anyhow::Result<((u64, u64), (u64, u64), (u64, u64))> { let (seal_pre_commit_phase1_measurement_cpu_time, seal_pre_commit_phase1_measurement_wall_time): (u64, u64) = if skip_precommit_phase1 { // generate no-op measurements @@ -109,28 +111,45 @@ fn run_pre_commit_phases( } else { // Create files for the staged and sealed sectors. let staged_file_path = cache_dir.join(STAGED_FILE); - let mut staged_file = OpenOptions::new().read(true).write(true).create(true).open(&staged_file_path)?; - info!("*** Created staged file"); + let mut staged_file = if skip_staging { + info!("*** Re-using staged file"); + OpenOptions::new().read(true).write(true).open(&staged_file_path) + } else { + info!("*** Creating staged file"); + OpenOptions::new().read(true).write(true).create(true).open(&staged_file_path) + }?; let sealed_file_path = cache_dir.join(SEALED_FILE); - let _sealed_file = OpenOptions::new().read(true).write(true).create(true).open(&sealed_file_path)?; - info!("*** Created sealed file"); + let _sealed_file = if skip_staging { + info!("*** Re-using sealed file"); + OpenOptions::new().read(true).write(true).open(&sealed_file_path) + } else { + info!("*** Creating sealed file"); + OpenOptions::new().read(true).write(true).create(true).open(&sealed_file_path) + }?; let sector_size_unpadded_bytes_amount = UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size)); - // Generate the data from which we will create a replica, we will then prove the continued - // storage of that replica using the PoSt. - let piece_bytes: Vec = (0..usize::from(sector_size_unpadded_bytes_amount)) - .map(|_| rand::random::()) - .collect(); - let piece_file_path = cache_dir.join(PIECE_FILE); - let mut piece_file = OpenOptions::new().read(true).write(true).create(true).open(&piece_file_path)?; - info!("*** Created piece file"); - piece_file.write_all(&piece_bytes)?; - piece_file.sync_all()?; - piece_file.seek(SeekFrom::Start(0))?; + let mut piece_file = if skip_staging { + info!("*** Re-using piece file"); + OpenOptions::new().read(true).write(true).open(&piece_file_path)? + } else { + // Generate the data from which we will create a replica, we will then prove the continued + // storage of that replica using the PoSt. + let piece_bytes: Vec = (0..usize::from(sector_size_unpadded_bytes_amount)) + .map(|_| rand::random::()) + .collect(); + + info!("*** Created piece file"); + let mut piece_file = OpenOptions::new().read(true).write(true).create(true).open(&piece_file_path)?; + piece_file.write_all(&piece_bytes)?; + piece_file.sync_all()?; + piece_file.seek(SeekFrom::Start(0))?; + + piece_file + }; let piece_info = generate_piece_commitment(&mut piece_file, sector_size_unpadded_bytes_amount)?; @@ -162,6 +181,30 @@ fn run_pre_commit_phases( .expect("failed in seal_pre_commit_phase1"); let precommit_phase1_output = seal_pre_commit_phase1_measurement.return_value; + if test_resume { + // Locate the last layer and delete it. + // + // NOTE: We should select a random layer to determine if + // the code will properly resume and re-write later layer + // files that exist beyond the missing one. + let mut layers = Vec::new(); + for entry in std::fs::read_dir(&cache_dir)? { + let cur = entry?; + let entry_path = cur.path(); + let entry_str = entry_path.to_str().expect("failed to get string from path"); + if entry_str.contains("data-layer") { + layers.push(entry_path.clone()); + } + } + + assert!(!layers.is_empty()); + // Delete the last layer only. + info!("Test resume requested. Removing last layer {:?}", layers[layers.len() - 1]); + std::fs::remove_file(&layers[layers.len() - 1])?; + + return run_pre_commit_phases::(sector_size, cache_dir, skip_precommit_phase1, skip_precommit_phase2, false, true); + } + // Persist piece_infos here let piece_infos_path = cache_dir.join(PIECE_INFOS_FILE); let mut f = File::create(&piece_infos_path) @@ -291,6 +334,7 @@ fn run_pre_commit_phases( )) } +#[allow(clippy::too_many_arguments)] pub fn run_window_post_bench( sector_size: u64, cache_dir: PathBuf, @@ -299,6 +343,7 @@ pub fn run_window_post_bench( skip_precommit_phase2: bool, skip_commit_phase1: bool, skip_commit_phase2: bool, + test_resume: bool, ) -> anyhow::Result<()> { let ( (seal_pre_commit_phase1_cpu_time_ms, seal_pre_commit_phase1_wall_time_ms), @@ -316,6 +361,8 @@ pub fn run_window_post_bench( cache_dir.clone(), skip_precommit_phase1, skip_precommit_phase2, + test_resume, + false, // skip staging ) }?; @@ -524,6 +571,7 @@ pub fn run_window_post_bench( Ok(()) } +#[allow(clippy::too_many_arguments)] pub fn run( sector_size: usize, cache: String, @@ -532,8 +580,9 @@ pub fn run( skip_precommit_phase2: bool, skip_commit_phase1: bool, skip_commit_phase2: bool, + test_resume: bool, ) -> anyhow::Result<()> { - info!("Benchy Window PoSt: sector-size={}, preserve_cache={}, skip_precommit_phase1={}, skip_precommit_phase2={}, skip_commit_phase1={}, skip_commit_phase2={}", sector_size, preserve_cache, skip_precommit_phase1, skip_precommit_phase2, skip_commit_phase1, skip_commit_phase2); + info!("Benchy Window PoSt: sector-size={}, preserve_cache={}, skip_precommit_phase1={}, skip_precommit_phase2={}, skip_commit_phase1={}, skip_commit_phase2={}, test_resume={}", sector_size, preserve_cache, skip_precommit_phase1, skip_precommit_phase2, skip_commit_phase1, skip_commit_phase2, test_resume); let cache_dir_specified = !cache.is_empty(); if skip_precommit_phase1 || skip_precommit_phase2 || skip_commit_phase1 || skip_commit_phase2 { @@ -573,5 +622,6 @@ pub fn run( skip_precommit_phase2, skip_commit_phase1, skip_commit_phase2, + test_resume, ) } diff --git a/storage-proofs/porep/src/stacked/vanilla/proof.rs b/storage-proofs/porep/src/stacked/vanilla/proof.rs index 76f764e59..451f375cc 100644 --- a/storage-proofs/porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/proof.rs @@ -502,7 +502,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr None }; - for (layer, layer_config) in (1..=layers).zip(layer_configs.into_iter()) { + for (layer, layer_config) in (1..=layers).zip(layer_configs) { info!("generating layer: {}", layer); if let Some(ref mut cache) = cache { @@ -2190,12 +2190,12 @@ mod tests { let unused_layer_challenges = LayerChallenges::new(layers, 0); - let (labels, _) = StackedDrg::< + let labels = StackedDrg::< // Although not generally correct for every size, the hasher shape is not used, // so for purposes of testing label creation, it is safe to supply a dummy. DiskTree, Sha256Hasher, - >::generate_labels( + >::generate_labels_for_decoding( &graph, &unused_layer_challenges, &::Domain::try_from_bytes(&replica_id).unwrap(),