From 6e4a3c52f3f6b5f9d4d3efe6323dd99ab7ca7bc8 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 22 Nov 2021 11:47:19 +1000 Subject: [PATCH 01/29] Add a missing feature dependency: download_params -> directories --- zcash_proofs/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index f43b4a5eb9..e61de507c0 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -40,7 +40,7 @@ pprof = { version = "0.8", features = ["criterion", "flamegraph"] } # MSRV 1.56 [features] default = ["local-prover", "multicore"] bundled-prover = ["wagyu-zcash-parameters"] -download-params = ["minreq"] +download-params = ["minreq", "directories"] local-prover = ["directories"] multicore = ["bellman/multicore"] From d79e9d810ff15b07d271d91155bdbd1f4b434528 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 22 Nov 2021 11:48:46 +1000 Subject: [PATCH 02/29] Make the parameter file names public --- zcash_proofs/src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index a4688d24f0..bc1fa1fb85 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -34,10 +34,14 @@ pub mod sprout; pub mod prover; // Circuit names -#[cfg(feature = "local-prover")] -const SAPLING_SPEND_NAME: &str = "sapling-spend.params"; -#[cfg(feature = "local-prover")] -const SAPLING_OUTPUT_NAME: &str = "sapling-output.params"; + +/// The sapling spend parameters file name. +#[cfg(any(feature = "local-prover", feature = "download-params"))] +pub const SAPLING_SPEND_NAME: &str = "sapling-spend.params"; + +/// The sapling output parameters file name. +#[cfg(any(feature = "local-prover", feature = "download-params"))] +pub const SAPLING_OUTPUT_NAME: &str = "sapling-output.params"; // Circuit hashes const SAPLING_SPEND_HASH: &str = "8270785a1a0d0bc77196f000ee6d221c9c9894f55307bd9357c3f0105d31ca63991ab91324160d8f53e2bbd3c2633a6eb8bdf5205d822e7f3f73edac51b2b70c"; From 38943266b3e9753d460f55c97ca5298ee00b7572 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 22 Nov 2021 11:50:50 +1000 Subject: [PATCH 03/29] Download sprout parameters in-memory --- zcash_proofs/src/lib.rs | 90 ++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index bc1fa1fb85..b6a9be7bda 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -43,6 +43,10 @@ pub const SAPLING_SPEND_NAME: &str = "sapling-spend.params"; #[cfg(any(feature = "local-prover", feature = "download-params"))] pub const SAPLING_OUTPUT_NAME: &str = "sapling-output.params"; +/// The sprout parameters file name. +#[cfg(any(feature = "local-prover", feature = "download-params"))] +pub const SPROUT_NAME: &str = "sprout-groth16.params"; + // Circuit hashes const SAPLING_SPEND_HASH: &str = "8270785a1a0d0bc77196f000ee6d221c9c9894f55307bd9357c3f0105d31ca63991ab91324160d8f53e2bbd3c2633a6eb8bdf5205d822e7f3f73edac51b2b70c"; const SAPLING_OUTPUT_HASH: &str = "657e3d38dbb5cb5e7dd2970e8b03d69b4787dd907285b5a7f0790dcc8072f60bf593b32cc2d1c030e00ff5ae64bf84c5c3beb84ddc841d48264b4a171744d028"; @@ -70,49 +74,63 @@ pub fn default_params_folder() -> Option { #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] pub fn download_parameters() -> Result<(), minreq::Error> { + download_sapling_parameters() +} + +/// Download the Zcash Sprout parameters, check their has, and store them in the default location. +/// +/// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. +#[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +pub fn download_sprout_parameters() -> Result<(), minreq::Error> { + fetch_params(SPROUT_NAME, SPROUT_HASH)?; + + Ok(()) +} + +/// Download the specified parameters, check their hash, and store them in the default location. +/// +/// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. +#[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +fn fetch_params(name: &str, expected_hash: &str) -> Result<(), minreq::Error> { + use std::io::Write; + // Ensure that the default Zcash parameters location exists. let params_dir = default_params_folder().ok_or_else(|| { io::Error::new(io::ErrorKind::Other, "Could not load default params folder") })?; std::fs::create_dir_all(¶ms_dir)?; - let fetch_params = |name: &str, expected_hash: &str| -> Result<(), minreq::Error> { - use std::io::Write; - - // Download the parts directly (Sapling parameters are small enough for this). - let part_1 = minreq::get(format!("{}/{}.part.1", DOWNLOAD_URL, name)).send()?; - let part_2 = minreq::get(format!("{}/{}.part.2", DOWNLOAD_URL, name)).send()?; - - // Verify parameter file hash. - let hash = blake2b_simd::State::new() - .update(part_1.as_bytes()) - .update(part_2.as_bytes()) - .finalize() - .to_hex(); - if &hash != expected_hash { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "{} failed validation (expected: {}, actual: {}, fetched {} bytes)", - name, - expected_hash, - hash, - part_1.as_bytes().len() + part_2.as_bytes().len() - ), - ) - .into()); - } - - // Write parameter file. - let mut f = File::create(params_dir.join(name))?; - f.write_all(part_1.as_bytes())?; - f.write_all(part_2.as_bytes())?; - Ok(()) - }; - - fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH)?; - fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH)?; + // Download the parts directly + // TODO: Sapling parameters are small enough for this, but Sprout parameters are ~720 MB. + let part_1 = minreq::get(format!("{}/{}.part.1", DOWNLOAD_URL, name)).send()?; + let part_2 = minreq::get(format!("{}/{}.part.2", DOWNLOAD_URL, name)).send()?; + + // Verify parameter file hash. + let hash = blake2b_simd::State::new() + .update(part_1.as_bytes()) + .update(part_2.as_bytes()) + .finalize() + .to_hex(); + if &hash != expected_hash { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} failed validation (expected: {}, actual: {}, fetched {} bytes)", + name, + expected_hash, + hash, + part_1.as_bytes().len() + part_2.as_bytes().len() + ), + ) + .into()); + } + // Write parameter file. + let mut f = File::create(params_dir.join(name))?; + f.write_all(part_1.as_bytes())?; + f.write_all(part_2.as_bytes())?; Ok(()) } From 5b3d4195326543d1399dd54e1d6b9a09d13baf8b Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 22 Nov 2021 11:51:22 +1000 Subject: [PATCH 04/29] Add download_sapling_parameters and deprecate download_parameters This avoids confusion between sprout and sapling downloads, while maintaining backward compatibility. --- zcash_proofs/src/lib.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index b6a9be7bda..1f316cbae6 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -68,15 +68,31 @@ pub fn default_params_folder() -> Option { }) } -/// Download the Zcash Sapling parameters, storing them in the default location. +/// Download the Zcash Sapling parameters, check their hash, and store them in the default location. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +#[deprecated( + since = "0.6.0", + note = "please replace with `download_sapling_parameters`, and add `download_sprout_parameters` if needed" +)] pub fn download_parameters() -> Result<(), minreq::Error> { download_sapling_parameters() } +/// Download the Zcash Sapling parameters, storing them in the default location. +/// +/// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. +#[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +pub fn download_sapling_parameters() -> Result<(), minreq::Error> { + fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH)?; + fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH)?; + + Ok(()) +} + /// Download the Zcash Sprout parameters, check their has, and store them in the default location. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. From 0abeac0a8c372c2da5025ca1805b48025d8d4ae6 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 05:27:26 +1000 Subject: [PATCH 05/29] Tweak deprecation warning --- zcash_proofs/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 1f316cbae6..9e6eb8eedb 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -75,7 +75,7 @@ pub fn default_params_folder() -> Option { #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] #[deprecated( since = "0.6.0", - note = "please replace with `download_sapling_parameters`, and add `download_sprout_parameters` if needed" + note = "please replace with `download_sapling_parameters`, and use `download_sprout_parameters` if needed" )] pub fn download_parameters() -> Result<(), minreq::Error> { download_sapling_parameters() From cbc377017259527890ff80545a304c87840e97ba Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 05:27:55 +1000 Subject: [PATCH 06/29] Download a single file, rather than parts This is more efficient, because TCP adjusts its transfer speed in the first ~20 seconds of each new connection. --- zcash_proofs/src/lib.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 9e6eb8eedb..53aed969d3 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -118,15 +118,14 @@ fn fetch_params(name: &str, expected_hash: &str) -> Result<(), minreq::Error> { })?; std::fs::create_dir_all(¶ms_dir)?; - // Download the parts directly + // Download the whole file into RAM // TODO: Sapling parameters are small enough for this, but Sprout parameters are ~720 MB. - let part_1 = minreq::get(format!("{}/{}.part.1", DOWNLOAD_URL, name)).send()?; - let part_2 = minreq::get(format!("{}/{}.part.2", DOWNLOAD_URL, name)).send()?; + let params_file = format!("{}/{}", DOWNLOAD_URL, name); + let params_data = minreq::get(params_file).send()?; // Verify parameter file hash. let hash = blake2b_simd::State::new() - .update(part_1.as_bytes()) - .update(part_2.as_bytes()) + .update(params_data.as_bytes()) .finalize() .to_hex(); if &hash != expected_hash { @@ -137,7 +136,7 @@ fn fetch_params(name: &str, expected_hash: &str) -> Result<(), minreq::Error> { name, expected_hash, hash, - part_1.as_bytes().len() + part_2.as_bytes().len() + params_data.as_bytes().len(), ), ) .into()); @@ -145,8 +144,7 @@ fn fetch_params(name: &str, expected_hash: &str) -> Result<(), minreq::Error> { // Write parameter file. let mut f = File::create(params_dir.join(name))?; - f.write_all(part_1.as_bytes())?; - f.write_all(part_2.as_bytes())?; + f.write_all(params_data.as_bytes())?; Ok(()) } From a46b2bf05769390ffa35b324aa38e0b2cd864ac6 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 05:42:18 +1000 Subject: [PATCH 07/29] Only download files if needed, but always check the hashes Also return the downloaded file paths. --- zcash_proofs/src/lib.rs | 115 ++++++++++++++++++++++++++++------------ 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 53aed969d3..8993529812 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -55,6 +55,16 @@ const SPROUT_HASH: &str = "e9b238411bd6c0ec4791e9d04245ec350c9c5744f5610dfcce436 #[cfg(feature = "download-params")] const DOWNLOAD_URL: &str = "https://download.z.cash/downloads"; +/// The paths to the Sapling parameter files. +#[cfg(feature = "download-params")] +pub struct SaplingParameterPaths { + /// The path to the Sapling spend parameter file. + pub spend: PathBuf, + + /// The path to the Sapling output parameter file. + pub output: PathBuf, +} + /// Returns the default folder that the Zcash proving parameters are located in. #[cfg(feature = "directories")] #[cfg_attr(docsrs, doc(cfg(feature = "directories")))] @@ -68,7 +78,8 @@ pub fn default_params_folder() -> Option { }) } -/// Download the Zcash Sapling parameters, check their hash, and store them in the default location. +/// Download the Zcash Sapling parameters if needed, and store them in the default location. +/// Always checks the hashes of the files, even if they didn't need to be downloaded. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. #[cfg(feature = "download-params")] @@ -78,39 +89,44 @@ pub fn default_params_folder() -> Option { note = "please replace with `download_sapling_parameters`, and use `download_sprout_parameters` if needed" )] pub fn download_parameters() -> Result<(), minreq::Error> { - download_sapling_parameters() + download_sapling_parameters().map(|_sapling_paths| ()) } -/// Download the Zcash Sapling parameters, storing them in the default location. +/// Download the Zcash Sapling parameters if needed, and store them in the default location. +/// Always checks the hashes of the files, even if they didn't need to be downloaded. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. +/// +/// Returns the paths to the downloaded files. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] -pub fn download_sapling_parameters() -> Result<(), minreq::Error> { - fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH)?; - fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH)?; +pub fn download_sapling_parameters() -> Result { + let spend = fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH)?; + let output = fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH)?; - Ok(()) + Ok(SaplingParameterPaths { spend, output }) } -/// Download the Zcash Sprout parameters, check their has, and store them in the default location. +/// Download the Zcash Sprout parameters if needed, and store them in the default location. +/// Always checks the hash of the file, even if it didn't need to be downloaded. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. +/// +/// Returns the path to the downloaded file. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] -pub fn download_sprout_parameters() -> Result<(), minreq::Error> { - fetch_params(SPROUT_NAME, SPROUT_HASH)?; - - Ok(()) +pub fn download_sprout_parameters() -> Result { + fetch_params(SPROUT_NAME, SPROUT_HASH) } -/// Download the specified parameters, check their hash, and store them in the default location. +/// Download the specified parameters if needed, and store them in the default location. +/// Always checks the hash of the file, even if it didn't need to be downloaded. /// -/// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. +/// Returns the path to the downloaded file. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] -fn fetch_params(name: &str, expected_hash: &str) -> Result<(), minreq::Error> { - use std::io::Write; +fn fetch_params(name: &str, expected_hash: &str) -> Result { + use std::{fs, io::Write}; // Ensure that the default Zcash parameters location exists. let params_dir = default_params_folder().ok_or_else(|| { @@ -118,34 +134,63 @@ fn fetch_params(name: &str, expected_hash: &str) -> Result<(), minreq::Error> { })?; std::fs::create_dir_all(¶ms_dir)?; - // Download the whole file into RAM - // TODO: Sapling parameters are small enough for this, but Sprout parameters are ~720 MB. - let params_file = format!("{}/{}", DOWNLOAD_URL, name); - let params_data = minreq::get(params_file).send()?; + let params_path = params_dir.join(name); + + // Download parameters if needed. + // TODO: use try_exists when it stabilises, to exit early on permissions errors (#83186) + if !params_path.exists() { + // Download the whole file into RAM + // TODO: Sapling parameters are small enough for this, but Sprout parameters are ~720 MB. + let params_url = format!("{}/{}", DOWNLOAD_URL, name); + let params_data = minreq::get(¶ms_url).send()?; + + verify_hash(params_data.as_bytes(), expected_hash, name, ¶ms_url)?; + + // Write parameter file. + let mut f = File::create(¶ms_path)?; + f.write_all(params_data.as_bytes())?; + } else { + let params_data = fs::read(¶ms_path)?; + + verify_hash( + ¶ms_data, + expected_hash, + name, + ¶ms_path.to_string_lossy(), + )?; + } + + Ok(params_path) +} + +/// Check if the `data` Blake2b hash matches `expected_hash`. +/// +/// Returns an error containing `name` and `source` on failure. +#[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +fn verify_hash( + data: &[u8], + expected_hash: &str, + name: &str, + source: &str, +) -> Result<(), io::Error> { + let hash = blake2b_simd::State::new().update(data).finalize().to_hex(); - // Verify parameter file hash. - let hash = blake2b_simd::State::new() - .update(params_data.as_bytes()) - .finalize() - .to_hex(); if &hash != expected_hash { - return Err(io::Error::new( + Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "{} failed validation (expected: {}, actual: {}, fetched {} bytes)", + "{} failed validation (expected: {}, actual: {}, fetched {} bytes from {:?})", name, expected_hash, hash, - params_data.as_bytes().len(), + data.len(), + source, ), - ) - .into()); + )) + } else { + Ok(()) } - - // Write parameter file. - let mut f = File::create(params_dir.join(name))?; - f.write_all(params_data.as_bytes())?; - Ok(()) } pub struct ZcashParameters { From 59bf5878f0a9126edee6901bae6463958a187d9c Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 06:31:27 +1000 Subject: [PATCH 08/29] Allow the caller to specify a response timeout --- zcash_proofs/src/lib.rs | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 8993529812..ddf55c13a2 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -89,7 +89,7 @@ pub fn default_params_folder() -> Option { note = "please replace with `download_sapling_parameters`, and use `download_sprout_parameters` if needed" )] pub fn download_parameters() -> Result<(), minreq::Error> { - download_sapling_parameters().map(|_sapling_paths| ()) + download_sapling_parameters(None).map(|_sapling_paths| ()) } /// Download the Zcash Sapling parameters if needed, and store them in the default location. @@ -97,12 +97,17 @@ pub fn download_parameters() -> Result<(), minreq::Error> { /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. /// +/// Use `timeout` to set a timeout in seconds for each file download. +/// If `timeout` is `None`, a timeout can be set using the `MINREQ_TIMEOUT` environmental variable. +/// /// Returns the paths to the downloaded files. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] -pub fn download_sapling_parameters() -> Result { - let spend = fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH)?; - let output = fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH)?; +pub fn download_sapling_parameters( + timeout: Option, +) -> Result { + let spend = fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH, timeout)?; + let output = fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH, timeout)?; Ok(SaplingParameterPaths { spend, output }) } @@ -112,11 +117,14 @@ pub fn download_sapling_parameters() -> Result Result { - fetch_params(SPROUT_NAME, SPROUT_HASH) +pub fn download_sprout_parameters(timeout: Option) -> Result { + fetch_params(SPROUT_NAME, SPROUT_HASH, timeout) } /// Download the specified parameters if needed, and store them in the default location. @@ -125,7 +133,11 @@ pub fn download_sprout_parameters() -> Result { /// Returns the path to the downloaded file. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] -fn fetch_params(name: &str, expected_hash: &str) -> Result { +fn fetch_params( + name: &str, + expected_hash: &str, + timeout: Option, +) -> Result { use std::{fs, io::Write}; // Ensure that the default Zcash parameters location exists. @@ -139,10 +151,16 @@ fn fetch_params(name: &str, expected_hash: &str) -> Result Date: Wed, 24 Nov 2021 08:08:30 +1000 Subject: [PATCH 09/29] Stream downloads from server to disk --- zcash_proofs/src/lib.rs | 139 +++++++++++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 39 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index ddf55c13a2..5af815151b 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -138,7 +138,7 @@ fn fetch_params( expected_hash: &str, timeout: Option, ) -> Result { - use std::{fs, io::Write}; + use std::io::BufWriter; // Ensure that the default Zcash parameters location exists. let params_dir = default_params_folder().ok_or_else(|| { @@ -151,27 +151,46 @@ fn fetch_params( // Download parameters if needed. // TODO: use try_exists when it stabilises, to exit early on permissions errors (#83186) if !params_path.exists() { - let params_url = format!("{}/{}", DOWNLOAD_URL, name); - let mut params_data = minreq::get(¶ms_url); + // Fail early if the directory isn't writeable. + let new_params_file = File::create(¶ms_path)?; + let new_params_file = BufWriter::with_capacity(1024 * 1024, new_params_file); + // Set up the download request. + let params_url = format!("{}/{}", DOWNLOAD_URL, name); + let mut params_download = minreq::get(¶ms_url); if let Some(timeout) = timeout { - params_data = params_data.with_timeout(timeout); + params_download = params_download.with_timeout(timeout); } - // Download the whole file into RAM - // TODO: Sapling parameters are small enough for this, but Sprout parameters are ~720 MB. - let params_data = params_data.send()?; - - verify_hash(params_data.as_bytes(), expected_hash, name, ¶ms_url)?; + // Download the response and write it to a new file, + // verifying the hash as bytes are read. + let params_download = params_download.send_lazy()?; + let params_download = ResponseLazyReader(params_download); + let params_download = BufReader::with_capacity(1024 * 1024, params_download); + let params_download = hashreader::HashReader::new(params_download); - // Write parameter file. - let mut f = File::create(¶ms_path)?; - f.write_all(params_data.as_bytes())?; + verify_hash( + params_download, + new_params_file, + expected_hash, + name, + ¶ms_url, + )?; } else { - let params_data = fs::read(¶ms_path)?; + // TODO: avoid reading the files twice + // Either: + // - return Ok if the paths exist (we might want to check file sizes), or + // - always load and return the parameters, for newly downloaded and existing files. + + // Read the file to verify the hash, + // discarding bytes after they're hashed. + let params_file = File::open(¶ms_path)?; + let params_file = BufReader::with_capacity(1024 * 1024, params_file); + let params_file = hashreader::HashReader::new(params_file); verify_hash( - ¶ms_data, + params_file, + io::sink(), expected_hash, name, ¶ms_path.to_string_lossy(), @@ -181,33 +200,35 @@ fn fetch_params( Ok(params_path) } -/// Check if the `data` Blake2b hash matches `expected_hash`. -/// -/// Returns an error containing `name` and `source` on failure. #[cfg(feature = "download-params")] -#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] -fn verify_hash( - data: &[u8], - expected_hash: &str, - name: &str, - source: &str, -) -> Result<(), io::Error> { - let hash = blake2b_simd::State::new().update(data).finalize().to_hex(); +struct ResponseLazyReader(minreq::ResponseLazy); - if &hash != expected_hash { - Err(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "{} failed validation (expected: {}, actual: {}, fetched {} bytes from {:?})", - name, - expected_hash, - hash, - data.len(), - source, - ), - )) - } else { - Ok(()) +#[cfg(feature = "download-params")] +impl io::Read for ResponseLazyReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + // Zero-sized buffer. This should never happen. + if buf.len() == 0 { + return Ok(0); + } + + // minreq has a very limited lazy reading interface. + match &mut self.0.next() { + // Read one byte into the buffer. + // We ignore the expected length, because we have no way of telling the BufReader. + Some(Ok((byte, _length))) => { + buf[0] = *byte; + Ok(1) + } + + // Reading failed. + Some(Err(error)) => Err(io::Error::new( + io::ErrorKind::Other, + format!("download failed: {:?}", error), + )), + + // Finished reading. + None => Ok(0), + } } } @@ -305,3 +326,43 @@ pub fn parse_parameters( sprout_vk, } } + +/// Check if the Blake2b hash from `hash_reader` matches `expected_hash`, +/// while streaming from `data` into `sink`. +/// +/// `hash_reader` can be used to partially read `data`, +/// before verifying the hash using this function. +/// +/// Returns an error containing `name` and `params_source` on failure. +fn verify_hash( + mut hash_reader: hashreader::HashReader, + mut sink: W, + expected_hash: &str, + name: &str, + params_source: &str, +) -> Result<(), io::Error> { + let read_result = io::copy(&mut hash_reader, &mut sink); + + if let Err(read_error) = read_result { + return Err(io::Error::new( + read_error.kind(), + format!( + "{} failed reading: {:?}, error: {:?}", + name, params_source, read_error, + ), + )); + } + + let hash = hash_reader.into_hash(); + if hash != expected_hash { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} failed validation: expected: {}, actual: {}, from: {:?}", + name, expected_hash, hash, params_source, + ), + )); + } + + Ok(()) +} From 19df7edaab20d7a08c084964d779bcab8f3ebe8c Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 08:09:03 +1000 Subject: [PATCH 10/29] Refactor file loads to use the same verifying function as downloads --- zcash_proofs/src/lib.rs | 52 ++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 5af815151b..aeba6b3b8b 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -289,28 +289,38 @@ pub fn parse_parameters( // want to read it, though, so that the BLAKE2b computed afterward is consistent // with `b2sum` on the files. let mut sink = io::sink(); - io::copy(&mut spend_fs, &mut sink) - .expect("couldn't finish reading Sapling spend parameter file"); - io::copy(&mut output_fs, &mut sink) - .expect("couldn't finish reading Sapling output parameter file"); - if let Some(mut sprout_fs) = sprout_fs.as_mut() { - io::copy(&mut sprout_fs, &mut sink) - .expect("couldn't finish reading Sprout groth16 parameter file"); - } - - if spend_fs.into_hash() != SAPLING_SPEND_HASH { - panic!("Sapling spend parameter file is not correct, please clean your `~/.zcash-params/` and re-run `fetch-params`."); - } - if output_fs.into_hash() != SAPLING_OUTPUT_HASH { - panic!("Sapling output parameter file is not correct, please clean your `~/.zcash-params/` and re-run `fetch-params`."); - } - - if sprout_fs - .map(|fs| fs.into_hash() != SPROUT_HASH) - .unwrap_or(false) - { - panic!("Sprout groth16 parameter file is not correct, please clean your `~/.zcash-params/` and re-run `fetch-params`."); + // TODO: use the correct paths for Windows and macOS + // use the actual file paths supplied by the caller + verify_hash( + spend_fs, + &mut sink, + SAPLING_SPEND_HASH, + SAPLING_SPEND_NAME, + "a file", + ) + .expect( + "Sapling spend parameter file is not correct, \ + please clean your `~/.zcash-params/` and re-run `fetch-params`.", + ); + + verify_hash( + output_fs, + &mut sink, + SAPLING_OUTPUT_HASH, + SAPLING_OUTPUT_NAME, + "a file", + ) + .expect( + "Sapling output parameter file is not correct, \ + please clean your `~/.zcash-params/` and re-run `fetch-params`.", + ); + + if let Some(sprout_fs) = sprout_fs { + verify_hash(sprout_fs, &mut sink, SPROUT_HASH, SPROUT_NAME, "a file").expect( + "Sprout groth16 parameter file is not correct, \ + please clean your `~/.zcash-params/` and re-run `fetch-params`.", + ); } // Prepare verifying keys From 6c67dc3ab9a91fce575d9f0b3de7cfa825a45f78 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 08:36:44 +1000 Subject: [PATCH 11/29] Add missing docs --- zcash_proofs/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index aeba6b3b8b..2d91af7bb7 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -81,6 +81,8 @@ pub fn default_params_folder() -> Option { /// Download the Zcash Sapling parameters if needed, and store them in the default location. /// Always checks the hashes of the files, even if they didn't need to be downloaded. /// +/// A timeout can be set using the `MINREQ_TIMEOUT` environmental variable. +/// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] From 33a83df2e6c8928959a17dcbde3e70e5c31823e7 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 12:41:31 +1000 Subject: [PATCH 12/29] Download parameters in parts, so we hit the Cloudflare cache --- zcash_proofs/src/lib.rs | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 2d91af7bb7..63a6c6f214 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -81,7 +81,7 @@ pub fn default_params_folder() -> Option { /// Download the Zcash Sapling parameters if needed, and store them in the default location. /// Always checks the hashes of the files, even if they didn't need to be downloaded. /// -/// A timeout can be set using the `MINREQ_TIMEOUT` environmental variable. +/// A download timeout can be set using the `MINREQ_TIMEOUT` environmental variable. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. #[cfg(feature = "download-params")] @@ -140,7 +140,7 @@ fn fetch_params( expected_hash: &str, timeout: Option, ) -> Result { - use std::io::BufWriter; + use std::io::{BufWriter, Read}; // Ensure that the default Zcash parameters location exists. let params_dir = default_params_folder().ok_or_else(|| { @@ -157,18 +157,29 @@ fn fetch_params( let new_params_file = File::create(¶ms_path)?; let new_params_file = BufWriter::with_capacity(1024 * 1024, new_params_file); - // Set up the download request. - let params_url = format!("{}/{}", DOWNLOAD_URL, name); - let mut params_download = minreq::get(¶ms_url); + // Set up the download requests. + // + // It's necessary for us to host these files in two parts, + // because of CloudFlare's maximum cached file size limit of 512 MB. + // The files must fit in the cache to prevent "denial of wallet" attacks. + let params_url_1 = format!("{}/{}.part.1", DOWNLOAD_URL, name); + // TODO: skip empty part.2 files when downloading sapling spend and sapling output + let params_url_2 = format!("{}/{}.part.2", DOWNLOAD_URL, name); + + let mut params_download_1 = minreq::get(¶ms_url_1); + let mut params_download_2 = minreq::get(¶ms_url_2); if let Some(timeout) = timeout { - params_download = params_download.with_timeout(timeout); + params_download_1 = params_download_1.with_timeout(timeout); + params_download_2 = params_download_2.with_timeout(timeout); } - // Download the response and write it to a new file, + // Download the responses and write them to a new file, // verifying the hash as bytes are read. - let params_download = params_download.send_lazy()?; - let params_download = ResponseLazyReader(params_download); - let params_download = BufReader::with_capacity(1024 * 1024, params_download); + let params_download_1 = ResponseLazyReader(params_download_1.send_lazy()?); + let params_download_2 = ResponseLazyReader(params_download_2.send_lazy()?); + + let params_download = + BufReader::with_capacity(1024 * 1024, params_download_1.chain(params_download_2)); let params_download = hashreader::HashReader::new(params_download); verify_hash( @@ -176,7 +187,7 @@ fn fetch_params( new_params_file, expected_hash, name, - ¶ms_url, + &format!("{} + {}", params_url_1, params_url_2), )?; } else { // TODO: avoid reading the files twice From 641c52f9af7da1792acba9f595208591b9083c6c Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 13:28:55 +1000 Subject: [PATCH 13/29] Add a byte count to verification errors --- zcash_proofs/src/hashreader.rs | 8 ++++++++ zcash_proofs/src/lib.rs | 12 ++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/zcash_proofs/src/hashreader.rs b/zcash_proofs/src/hashreader.rs index f8487b86ac..c8663a3d6b 100644 --- a/zcash_proofs/src/hashreader.rs +++ b/zcash_proofs/src/hashreader.rs @@ -5,6 +5,7 @@ use std::io::{self, Read}; pub struct HashReader { reader: R, hasher: State, + byte_count: usize, } impl HashReader { @@ -13,6 +14,7 @@ impl HashReader { HashReader { reader, hasher: State::new(), + byte_count: 0, } } @@ -27,6 +29,11 @@ impl HashReader { s } + + /// Return the number of bytes read so far. + pub fn byte_count(&self) -> usize { + self.byte_count + } } impl Read for HashReader { @@ -35,6 +42,7 @@ impl Read for HashReader { if bytes > 0 { self.hasher.update(&buf[0..bytes]); + self.byte_count += bytes; } Ok(bytes) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 63a6c6f214..326232810e 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -370,19 +370,23 @@ fn verify_hash( return Err(io::Error::new( read_error.kind(), format!( - "{} failed reading: {:?}, error: {:?}", - name, params_source, read_error, + "{} failed reading after {} bytes from {}, error: {:?}", + name, + params_source, + hash_reader.byte_count(), + read_error, ), )); } + let byte_count = hash_reader.byte_count(); let hash = hash_reader.into_hash(); if hash != expected_hash { return Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "{} failed validation: expected: {}, actual: {}, from: {:?}", - name, expected_hash, hash, params_source, + "{} failed validation: expected: {}, actual: {}, hashed {} bytes from {}", + name, expected_hash, hash, byte_count, params_source, ), )); } From 3f3116e151e132f0b036ecd72ed5f8397987c9d9 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 24 Nov 2021 14:08:39 +1000 Subject: [PATCH 14/29] Check file sizes to help debug parameter load failures --- zcash_proofs/src/lib.rs | 139 +++++++++++++++++++++++++++++++++++----- 1 file changed, 123 insertions(+), 16 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 326232810e..5d7fb9b1dc 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -52,6 +52,11 @@ const SAPLING_SPEND_HASH: &str = "8270785a1a0d0bc77196f000ee6d221c9c9894f55307bd const SAPLING_OUTPUT_HASH: &str = "657e3d38dbb5cb5e7dd2970e8b03d69b4787dd907285b5a7f0790dcc8072f60bf593b32cc2d1c030e00ff5ae64bf84c5c3beb84ddc841d48264b4a171744d028"; const SPROUT_HASH: &str = "e9b238411bd6c0ec4791e9d04245ec350c9c5744f5610dfcce4365d5ca49dfefd5054e371842b3f88fa1b9d7e8e075249b3ebabd167fa8b0f3161292d36c180a"; +// Circuit parameter file sizes +const SAPLING_SPEND_BYTES: u64 = 47958396; +const SAPLING_OUTPUT_BYTES: u64 = 3592860; +const SPROUT_BYTES: u64 = 725523612; + #[cfg(feature = "download-params")] const DOWNLOAD_URL: &str = "https://download.z.cash/downloads"; @@ -79,7 +84,7 @@ pub fn default_params_folder() -> Option { } /// Download the Zcash Sapling parameters if needed, and store them in the default location. -/// Always checks the hashes of the files, even if they didn't need to be downloaded. +/// Always checks the sizes and hashes of the files, even if they didn't need to be downloaded. /// /// A download timeout can be set using the `MINREQ_TIMEOUT` environmental variable. /// @@ -95,7 +100,7 @@ pub fn download_parameters() -> Result<(), minreq::Error> { } /// Download the Zcash Sapling parameters if needed, and store them in the default location. -/// Always checks the hashes of the files, even if they didn't need to be downloaded. +/// Always checks the sizes and hashes of the files, even if they didn't need to be downloaded. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. /// @@ -108,14 +113,24 @@ pub fn download_parameters() -> Result<(), minreq::Error> { pub fn download_sapling_parameters( timeout: Option, ) -> Result { - let spend = fetch_params(SAPLING_SPEND_NAME, SAPLING_SPEND_HASH, timeout)?; - let output = fetch_params(SAPLING_OUTPUT_NAME, SAPLING_OUTPUT_HASH, timeout)?; + let spend = fetch_params( + SAPLING_SPEND_NAME, + SAPLING_SPEND_HASH, + SAPLING_SPEND_BYTES, + timeout, + )?; + let output = fetch_params( + SAPLING_OUTPUT_NAME, + SAPLING_OUTPUT_HASH, + SAPLING_OUTPUT_BYTES, + timeout, + )?; Ok(SaplingParameterPaths { spend, output }) } /// Download the Zcash Sprout parameters if needed, and store them in the default location. -/// Always checks the hash of the file, even if it didn't need to be downloaded. +/// Always checks the size and hash of the file, even if it didn't need to be downloaded. /// /// This mirrors the behaviour of the `fetch-params.sh` script from `zcashd`. /// @@ -126,11 +141,11 @@ pub fn download_sapling_parameters( #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] pub fn download_sprout_parameters(timeout: Option) -> Result { - fetch_params(SPROUT_NAME, SPROUT_HASH, timeout) + fetch_params(SPROUT_NAME, SPROUT_HASH, SPROUT_BYTES, timeout) } /// Download the specified parameters if needed, and store them in the default location. -/// Always checks the hash of the file, even if it didn't need to be downloaded. +/// Always checks the size and hash of the file, even if it didn't need to be downloaded. /// /// Returns the path to the downloaded file. #[cfg(feature = "download-params")] @@ -138,6 +153,7 @@ pub fn download_sprout_parameters(timeout: Option) -> Result, ) -> Result { use std::io::{BufWriter, Read}; @@ -178,23 +194,35 @@ fn fetch_params( let params_download_1 = ResponseLazyReader(params_download_1.send_lazy()?); let params_download_2 = ResponseLazyReader(params_download_2.send_lazy()?); - let params_download = - BufReader::with_capacity(1024 * 1024, params_download_1.chain(params_download_2)); + // Limit the download size to avoid DoS. + let params_download = params_download_1 + .chain(params_download_2) + .take(expected_bytes); + let params_download = BufReader::with_capacity(1024 * 1024, params_download); let params_download = hashreader::HashReader::new(params_download); verify_hash( params_download, new_params_file, expected_hash, + expected_bytes, name, &format!("{} + {}", params_url_1, params_url_2), )?; } else { // TODO: avoid reading the files twice // Either: - // - return Ok if the paths exist (we might want to check file sizes), or + // - return Ok if the paths exist, or // - always load and return the parameters, for newly downloaded and existing files. + let file_path_string = params_path.to_string_lossy(); + + // Check the file size is correct before hashing large amounts of data. + verify_file_size(¶ms_path, expected_bytes, name, &file_path_string).expect( + "parameter file size is not correct, \ + please clean your Zcash parameters directory and re-run `fetch-params`.", + ); + // Read the file to verify the hash, // discarding bytes after they're hashed. let params_file = File::open(¶ms_path)?; @@ -205,8 +233,9 @@ fn fetch_params( params_file, io::sink(), expected_hash, + expected_bytes, name, - ¶ms_path.to_string_lossy(), + &file_path_string, )?; } @@ -253,11 +282,50 @@ pub struct ZcashParameters { pub sprout_vk: Option>, } +/// Load the specified parameters, checking the sizes and hashes of the files. +/// +/// Returns the loaded parameters. pub fn load_parameters( spend_path: &Path, output_path: &Path, sprout_path: Option<&Path>, ) -> ZcashParameters { + // Check the file sizes are correct before hashing large amounts of data. + verify_file_size( + spend_path, + SAPLING_SPEND_BYTES, + "sapling spend", + &spend_path.to_string_lossy(), + ) + .expect( + "parameter file size is not correct, \ + please clean your Zcash parameters directory and re-run `fetch-params`.", + ); + + verify_file_size( + output_path, + SAPLING_OUTPUT_BYTES, + "sapling output", + &output_path.to_string_lossy(), + ) + .expect( + "parameter file size is not correct, \ + please clean your Zcash parameters directory and re-run `fetch-params`.", + ); + + if let Some(sprout_path) = sprout_path { + verify_file_size( + sprout_path, + SPROUT_BYTES, + "sprout groth16", + &sprout_path.to_string_lossy(), + ) + .expect( + "parameter file size is not correct, \ + please clean your Zcash parameters directory and re-run `fetch-params`.", + ); + } + // Load from each of the paths let spend_fs = File::open(spend_path).expect("couldn't load Sapling spend parameters file"); let output_fs = File::open(output_path).expect("couldn't load Sapling output parameters file"); @@ -309,6 +377,7 @@ pub fn parse_parameters( spend_fs, &mut sink, SAPLING_SPEND_HASH, + SAPLING_SPEND_BYTES, SAPLING_SPEND_NAME, "a file", ) @@ -321,6 +390,7 @@ pub fn parse_parameters( output_fs, &mut sink, SAPLING_OUTPUT_HASH, + SAPLING_OUTPUT_BYTES, SAPLING_OUTPUT_NAME, "a file", ) @@ -330,9 +400,17 @@ pub fn parse_parameters( ); if let Some(sprout_fs) = sprout_fs { - verify_hash(sprout_fs, &mut sink, SPROUT_HASH, SPROUT_NAME, "a file").expect( + verify_hash( + sprout_fs, + &mut sink, + SPROUT_HASH, + SPROUT_BYTES, + SPROUT_NAME, + "a file", + ) + .expect( "Sprout groth16 parameter file is not correct, \ - please clean your `~/.zcash-params/` and re-run `fetch-params`.", + please clean your `~/.zcash-params/` and re-run `fetch-params`.", ); } @@ -350,6 +428,32 @@ pub fn parse_parameters( } } +/// Check if the size of the file at `params_path` matches `expected_bytes`, +/// using filesystem metadata. +/// +/// Returns an error containing `name` and `params_source` on failure. +fn verify_file_size( + params_path: &Path, + expected_bytes: u64, + name: &str, + params_source: &str, +) -> Result<(), io::Error> { + let file_size = std::fs::metadata(¶ms_path)?.len(); + + if file_size != expected_bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "{} failed validation: expected {} bytes, \ + actual {} bytes from {:?}", + name, expected_bytes, file_size, params_source, + ), + )); + } + + Ok(()) +} + /// Check if the Blake2b hash from `hash_reader` matches `expected_hash`, /// while streaming from `data` into `sink`. /// @@ -361,6 +465,7 @@ fn verify_hash( mut hash_reader: hashreader::HashReader, mut sink: W, expected_hash: &str, + expected_bytes: u64, name: &str, params_source: &str, ) -> Result<(), io::Error> { @@ -370,10 +475,11 @@ fn verify_hash( return Err(io::Error::new( read_error.kind(), format!( - "{} failed reading after {} bytes from {}, error: {:?}", + "{} failed reading after {} bytes, expected {} bytes from {:?}, error: {:?}", name, params_source, hash_reader.byte_count(), + expected_bytes, read_error, ), )); @@ -385,8 +491,9 @@ fn verify_hash( return Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "{} failed validation: expected: {}, actual: {}, hashed {} bytes from {}", - name, expected_hash, hash, byte_count, params_source, + "{} failed validation: expected: {} hashing {} bytes, \ + actual: {} hashing {} bytes from {:?}", + name, expected_hash, expected_bytes, hash, byte_count, params_source, ), )); } From 12ad78288e9ea76a33b75c14e1c89f91a2c4427d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 25 Nov 2021 07:30:19 +1000 Subject: [PATCH 15/29] Remove downloaded files on error (but leave existing files alone) --- zcash_proofs/src/lib.rs | 103 +++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 5d7fb9b1dc..b657bcf3ae 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -169,46 +169,19 @@ fn fetch_params( // Download parameters if needed. // TODO: use try_exists when it stabilises, to exit early on permissions errors (#83186) if !params_path.exists() { - // Fail early if the directory isn't writeable. - let new_params_file = File::create(¶ms_path)?; - let new_params_file = BufWriter::with_capacity(1024 * 1024, new_params_file); - - // Set up the download requests. - // - // It's necessary for us to host these files in two parts, - // because of CloudFlare's maximum cached file size limit of 512 MB. - // The files must fit in the cache to prevent "denial of wallet" attacks. - let params_url_1 = format!("{}/{}.part.1", DOWNLOAD_URL, name); - // TODO: skip empty part.2 files when downloading sapling spend and sapling output - let params_url_2 = format!("{}/{}.part.2", DOWNLOAD_URL, name); - - let mut params_download_1 = minreq::get(¶ms_url_1); - let mut params_download_2 = minreq::get(¶ms_url_2); - if let Some(timeout) = timeout { - params_download_1 = params_download_1.with_timeout(timeout); - params_download_2 = params_download_2.with_timeout(timeout); - } - - // Download the responses and write them to a new file, - // verifying the hash as bytes are read. - let params_download_1 = ResponseLazyReader(params_download_1.send_lazy()?); - let params_download_2 = ResponseLazyReader(params_download_2.send_lazy()?); - - // Limit the download size to avoid DoS. - let params_download = params_download_1 - .chain(params_download_2) - .take(expected_bytes); - let params_download = BufReader::with_capacity(1024 * 1024, params_download); - let params_download = hashreader::HashReader::new(params_download); - - verify_hash( - params_download, - new_params_file, + result = stream_params_downloads_to_disk( + params_path, + name, expected_hash, expected_bytes, - name, - &format!("{} + {}", params_url_1, params_url_2), - )?; + timeout, + ); + + // Remove the file on error, and return the download or hash error. + if result.is_err() { + let _ = std::fs::remove_file(params_path); + result?; + } } else { // TODO: avoid reading the files twice // Either: @@ -242,6 +215,60 @@ fn fetch_params( Ok(params_path) } +/// Download the specified parameters, stream them to `params_path`, and check their hash. +/// +/// Returns the path to the downloaded file. +#[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +fn stream_params_downloads_to_disk( + params_path: &Path, + name: &str, + expected_hash: &str, + expected_bytes: u64, + timeout: Option, +) -> Result { + // Fail early if the directory isn't writeable. + let new_params_file = File::create(¶ms_path)?; + let new_params_file = BufWriter::with_capacity(1024 * 1024, new_params_file); + + // Set up the download requests. + // + // It's necessary for us to host these files in two parts, + // because of CloudFlare's maximum cached file size limit of 512 MB. + // The files must fit in the cache to prevent "denial of wallet" attacks. + let params_url_1 = format!("{}/{}.part.1", DOWNLOAD_URL, name); + // TODO: skip empty part.2 files when downloading sapling spend and sapling output + let params_url_2 = format!("{}/{}.part.2", DOWNLOAD_URL, name); + + let mut params_download_1 = minreq::get(¶ms_url_1); + let mut params_download_2 = minreq::get(¶ms_url_2); + if let Some(timeout) = timeout { + params_download_1 = params_download_1.with_timeout(timeout); + params_download_2 = params_download_2.with_timeout(timeout); + } + + // Download the responses and write them to a new file, + // verifying the hash as bytes are read. + let params_download_1 = ResponseLazyReader(params_download_1.send_lazy()?); + let params_download_2 = ResponseLazyReader(params_download_2.send_lazy()?); + + // Limit the download size to avoid DoS. + let params_download = params_download_1 + .chain(params_download_2) + .take(expected_bytes); + let params_download = BufReader::with_capacity(1024 * 1024, params_download); + let params_download = hashreader::HashReader::new(params_download); + + verify_hash( + params_download, + new_params_file, + expected_hash, + expected_bytes, + name, + &format!("{} + {}", params_url_1, params_url_2), + )?; +} + #[cfg(feature = "download-params")] struct ResponseLazyReader(minreq::ResponseLazy); From fa901f88de63114021a3b07f6294911da9a2111d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 25 Nov 2021 07:31:03 +1000 Subject: [PATCH 16/29] Add a sprout and sapling download example --- zcash_proofs/Cargo.toml | 4 ++++ .../download-sprout-and-sapling-params.rs | 17 ++++++++++++++++ zcash_proofs/src/lib.rs | 20 ++++++++++--------- 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 zcash_proofs/examples/download-sprout-and-sapling-params.rs diff --git a/zcash_proofs/Cargo.toml b/zcash_proofs/Cargo.toml index e61de507c0..6925d05f27 100644 --- a/zcash_proofs/Cargo.toml +++ b/zcash_proofs/Cargo.toml @@ -60,5 +60,9 @@ required-features = ["directories"] name = "download-params" required-features = ["download-params"] +[[example]] +name = "download-sprout-and-sapling-params" +required-features = ["download-params"] + [badges] maintenance = { status = "actively-developed" } diff --git a/zcash_proofs/examples/download-sprout-and-sapling-params.rs b/zcash_proofs/examples/download-sprout-and-sapling-params.rs new file mode 100644 index 0000000000..25e120357e --- /dev/null +++ b/zcash_proofs/examples/download-sprout-and-sapling-params.rs @@ -0,0 +1,17 @@ +fn main() -> Result<(), minreq::Error> { + const DOWNLOAD_TIMEOUT_SECONDS: u64 = 3600; + + // Always do a download to /tmp, if compiled with `RUSTFLAGS="--cfg always_download"` + #[cfg(always_download)] + { + std::env::set_var("HOME", "/tmp"); + let _ = std::fs::remove_dir( + zcash_proofs::default_params_folder().expect("unexpected missing HOME env var"), + ); + } + + zcash_proofs::download_sapling_parameters(Some(DOWNLOAD_TIMEOUT_SECONDS))?; + zcash_proofs::download_sprout_parameters(Some(DOWNLOAD_TIMEOUT_SECONDS))?; + + Ok(()) +} diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index b657bcf3ae..6b81c46e9d 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -147,7 +147,7 @@ pub fn download_sprout_parameters(timeout: Option) -> Result, ) -> Result { - use std::io::{BufWriter, Read}; - // Ensure that the default Zcash parameters location exists. let params_dir = default_params_folder().ok_or_else(|| { io::Error::new(io::ErrorKind::Other, "Could not load default params folder") @@ -169,8 +167,8 @@ fn fetch_params( // Download parameters if needed. // TODO: use try_exists when it stabilises, to exit early on permissions errors (#83186) if !params_path.exists() { - result = stream_params_downloads_to_disk( - params_path, + let result = stream_params_downloads_to_disk( + ¶ms_path, name, expected_hash, expected_bytes, @@ -179,7 +177,7 @@ fn fetch_params( // Remove the file on error, and return the download or hash error. if result.is_err() { - let _ = std::fs::remove_file(params_path); + let _ = std::fs::remove_file(¶ms_path); result?; } } else { @@ -215,9 +213,9 @@ fn fetch_params( Ok(params_path) } -/// Download the specified parameters, stream them to `params_path`, and check their hash. +/// Download the specified parameter file, stream it to `params_path`, and check its hash. /// -/// Returns the path to the downloaded file. +/// See [`download_sapling_parameters`] for details. #[cfg(feature = "download-params")] #[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] fn stream_params_downloads_to_disk( @@ -226,7 +224,9 @@ fn stream_params_downloads_to_disk( expected_hash: &str, expected_bytes: u64, timeout: Option, -) -> Result { +) -> Result<(), minreq::Error> { + use std::io::{BufWriter, Read}; + // Fail early if the directory isn't writeable. let new_params_file = File::create(¶ms_path)?; let new_params_file = BufWriter::with_capacity(1024 * 1024, new_params_file); @@ -267,6 +267,8 @@ fn stream_params_downloads_to_disk( name, &format!("{} + {}", params_url_1, params_url_2), )?; + + Ok(()) } #[cfg(feature = "download-params")] From 3a005fdfbb08373559aaed89b35f0b7e37e36130 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 25 Nov 2021 10:50:39 +1000 Subject: [PATCH 17/29] Improve example downloaders --- zcash_proofs/examples/download-params.rs | 1 + .../download-sprout-and-sapling-params.rs | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/zcash_proofs/examples/download-params.rs b/zcash_proofs/examples/download-params.rs index b23cb75fae..6c9d97754d 100644 --- a/zcash_proofs/examples/download-params.rs +++ b/zcash_proofs/examples/download-params.rs @@ -1,3 +1,4 @@ fn main() -> Result<(), minreq::Error> { + #[allow(deprecated)] zcash_proofs::download_parameters() } diff --git a/zcash_proofs/examples/download-sprout-and-sapling-params.rs b/zcash_proofs/examples/download-sprout-and-sapling-params.rs index 25e120357e..ec4cdb5f11 100644 --- a/zcash_proofs/examples/download-sprout-and-sapling-params.rs +++ b/zcash_proofs/examples/download-sprout-and-sapling-params.rs @@ -1,16 +1,25 @@ fn main() -> Result<(), minreq::Error> { const DOWNLOAD_TIMEOUT_SECONDS: u64 = 3600; + #[allow(unused_mut, unused_assignments)] + let mut params_folder = + zcash_proofs::default_params_folder().expect("unexpected missing HOME env var"); + // Always do a download to /tmp, if compiled with `RUSTFLAGS="--cfg always_download"` #[cfg(always_download)] { std::env::set_var("HOME", "/tmp"); - let _ = std::fs::remove_dir( - zcash_proofs::default_params_folder().expect("unexpected missing HOME env var"), - ); + params_folder = + zcash_proofs::default_params_folder().expect("unexpected missing HOME env var"); + + println!("removing temporary parameters folder: {:?}", params_folder); + let _ = std::fs::remove_dir_all(¶ms_folder); } + println!("downloading sapling parameters to: {:?}", params_folder); zcash_proofs::download_sapling_parameters(Some(DOWNLOAD_TIMEOUT_SECONDS))?; + + println!("downloading sprout parameters to: {:?}", params_folder); zcash_proofs::download_sprout_parameters(Some(DOWNLOAD_TIMEOUT_SECONDS))?; Ok(()) From e50cce294cd53d8276b9e497eb504787363fb730 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 25 Nov 2021 10:51:16 +1000 Subject: [PATCH 18/29] Fix missing docs --- zcash_proofs/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 6b81c46e9d..e57e9d3a45 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -303,6 +303,7 @@ impl io::Read for ResponseLazyReader { } } +/// Zcash Sprout and Sapling groth16 circuit parameters. pub struct ZcashParameters { pub spend_params: Parameters, pub spend_vk: PreparedVerifyingKey, From 2737d6d99757af21f453c2bc52af85f3f1141c18 Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 25 Nov 2021 10:51:36 +1000 Subject: [PATCH 19/29] Launch the second download request once the first finishes --- zcash_proofs/src/lib.rs | 89 ++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index e57e9d3a45..d0f558d0f1 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -249,10 +249,11 @@ fn stream_params_downloads_to_disk( // Download the responses and write them to a new file, // verifying the hash as bytes are read. - let params_download_1 = ResponseLazyReader(params_download_1.send_lazy()?); - let params_download_2 = ResponseLazyReader(params_download_2.send_lazy()?); + let params_download_1 = ResponseLazyReader::from(params_download_1); + let params_download_2 = ResponseLazyReader::from(params_download_2); // Limit the download size to avoid DoS. + // This also avoids launching the second request, if the first request provides enough bytes. let params_download = params_download_1 .chain(params_download_2) .take(expected_bytes); @@ -271,34 +272,82 @@ fn stream_params_downloads_to_disk( Ok(()) } +/// A wrapper that implements [`io::Read`] on a [`minreq::ResponseLazy`]. #[cfg(feature = "download-params")] -struct ResponseLazyReader(minreq::ResponseLazy); +enum ResponseLazyReader { + Request(minreq::Request), + Response(minreq::ResponseLazy), + Complete(Result<(), String>), +} + +#[cfg(feature = "download-params")] +impl From for ResponseLazyReader { + fn from(request: minreq::Request) -> ResponseLazyReader { + ResponseLazyReader::Request(request) + } +} + +#[cfg(feature = "download-params")] +impl From for ResponseLazyReader { + fn from(response: minreq::ResponseLazy) -> ResponseLazyReader { + ResponseLazyReader::Response(response) + } +} #[cfg(feature = "download-params")] impl io::Read for ResponseLazyReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { + use ResponseLazyReader::*; + // Zero-sized buffer. This should never happen. - if buf.len() == 0 { + if buf.is_empty() { return Ok(0); } - // minreq has a very limited lazy reading interface. - match &mut self.0.next() { - // Read one byte into the buffer. - // We ignore the expected length, because we have no way of telling the BufReader. - Some(Ok((byte, _length))) => { - buf[0] = *byte; - Ok(1) + loop { + match self { + // Launch a lazy response for this request + Request(request) => match request.clone().send_lazy() { + Ok(response) => *self = Response(response), + Err(error) => { + let error = Err(format!("download request failed: {:?}", error)); + + *self = Complete(error); + } + }, + + // Read from the response + Response(response) => { + // minreq has a very limited lazy reading interface. + match &mut response.next() { + // Read one byte into the buffer. + // We ignore the expected length, because we have no way of telling the BufReader. + Some(Ok((byte, _length))) => { + buf[0] = *byte; + return Ok(1); + } + + // Reading failed. + Some(Err(error)) => { + let error = Err(format!("download response failed: {:?}", error)); + + *self = Complete(error); + } + + // Finished reading. + None => *self = Complete(Ok(())), + } + } + + Complete(result) => { + return match result { + // Return a zero-byte read for download success and EOF. + Ok(()) => Ok(0), + // Keep returning the download error, + Err(error) => Err(io::Error::new(io::ErrorKind::Other, error.clone())), + }; + } } - - // Reading failed. - Some(Err(error)) => Err(io::Error::new( - io::ErrorKind::Other, - format!("download failed: {:?}", error), - )), - - // Finished reading. - None => Ok(0), } } } From 5ece02d5dea5b29a84202ef1f075ce3d3346b31f Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 25 Nov 2021 11:00:28 +1000 Subject: [PATCH 20/29] Move the download Read impl into its own module --- zcash_proofs/src/downloadreader.rs | 79 +++++++++++++++++++++++++++ zcash_proofs/src/lib.rs | 88 ++---------------------------- 2 files changed, 85 insertions(+), 82 deletions(-) create mode 100644 zcash_proofs/src/downloadreader.rs diff --git a/zcash_proofs/src/downloadreader.rs b/zcash_proofs/src/downloadreader.rs new file mode 100644 index 0000000000..af408dae31 --- /dev/null +++ b/zcash_proofs/src/downloadreader.rs @@ -0,0 +1,79 @@ +//! [`io::Read`] implementations for [`minreq`]. + +use std::io; + +/// A wrapper that implements [`io::Read`] on a [`minreq::ResponseLazy`]. +pub enum ResponseLazyReader { + Request(minreq::Request), + Response(minreq::ResponseLazy), + Complete(Result<(), String>), +} + +impl From for ResponseLazyReader { + fn from(request: minreq::Request) -> ResponseLazyReader { + ResponseLazyReader::Request(request) + } +} + +impl From for ResponseLazyReader { + fn from(response: minreq::ResponseLazy) -> ResponseLazyReader { + ResponseLazyReader::Response(response) + } +} + +impl io::Read for ResponseLazyReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + use ResponseLazyReader::*; + + // Zero-sized buffer. This should never happen. + if buf.is_empty() { + return Ok(0); + } + + loop { + match self { + // Launch a lazy response for this request + Request(request) => match request.clone().send_lazy() { + Ok(response) => *self = Response(response), + Err(error) => { + let error = Err(format!("download request failed: {:?}", error)); + + *self = Complete(error); + } + }, + + // Read from the response + Response(response) => { + // minreq has a very limited lazy reading interface. + match &mut response.next() { + // Read one byte into the buffer. + // We ignore the expected length, because we have no way of telling the BufReader. + Some(Ok((byte, _length))) => { + buf[0] = *byte; + return Ok(1); + } + + // Reading failed. + Some(Err(error)) => { + let error = Err(format!("download response failed: {:?}", error)); + + *self = Complete(error); + } + + // Finished reading. + None => *self = Complete(Ok(())), + } + } + + Complete(result) => { + return match result { + // Return a zero-byte read for download success and EOF. + Ok(()) => Ok(0), + // Keep returning the download error, + Err(error) => Err(io::Error::new(io::ErrorKind::Other, error.clone())), + }; + } + } + } + } +} diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index d0f558d0f1..12268a14e1 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -33,6 +33,10 @@ pub mod sprout; )] pub mod prover; +#[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] +mod downloadreader; + // Circuit names /// The sapling spend parameters file name. @@ -249,8 +253,8 @@ fn stream_params_downloads_to_disk( // Download the responses and write them to a new file, // verifying the hash as bytes are read. - let params_download_1 = ResponseLazyReader::from(params_download_1); - let params_download_2 = ResponseLazyReader::from(params_download_2); + let params_download_1 = downloadreader::ResponseLazyReader::from(params_download_1); + let params_download_2 = downloadreader::ResponseLazyReader::from(params_download_2); // Limit the download size to avoid DoS. // This also avoids launching the second request, if the first request provides enough bytes. @@ -272,86 +276,6 @@ fn stream_params_downloads_to_disk( Ok(()) } -/// A wrapper that implements [`io::Read`] on a [`minreq::ResponseLazy`]. -#[cfg(feature = "download-params")] -enum ResponseLazyReader { - Request(minreq::Request), - Response(minreq::ResponseLazy), - Complete(Result<(), String>), -} - -#[cfg(feature = "download-params")] -impl From for ResponseLazyReader { - fn from(request: minreq::Request) -> ResponseLazyReader { - ResponseLazyReader::Request(request) - } -} - -#[cfg(feature = "download-params")] -impl From for ResponseLazyReader { - fn from(response: minreq::ResponseLazy) -> ResponseLazyReader { - ResponseLazyReader::Response(response) - } -} - -#[cfg(feature = "download-params")] -impl io::Read for ResponseLazyReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - use ResponseLazyReader::*; - - // Zero-sized buffer. This should never happen. - if buf.is_empty() { - return Ok(0); - } - - loop { - match self { - // Launch a lazy response for this request - Request(request) => match request.clone().send_lazy() { - Ok(response) => *self = Response(response), - Err(error) => { - let error = Err(format!("download request failed: {:?}", error)); - - *self = Complete(error); - } - }, - - // Read from the response - Response(response) => { - // minreq has a very limited lazy reading interface. - match &mut response.next() { - // Read one byte into the buffer. - // We ignore the expected length, because we have no way of telling the BufReader. - Some(Ok((byte, _length))) => { - buf[0] = *byte; - return Ok(1); - } - - // Reading failed. - Some(Err(error)) => { - let error = Err(format!("download response failed: {:?}", error)); - - *self = Complete(error); - } - - // Finished reading. - None => *self = Complete(Ok(())), - } - } - - Complete(result) => { - return match result { - // Return a zero-byte read for download success and EOF. - Ok(()) => Ok(0), - // Keep returning the download error, - Err(error) => Err(io::Error::new(io::ErrorKind::Other, error.clone())), - }; - } - } - } - } -} - /// Zcash Sprout and Sapling groth16 circuit parameters. pub struct ZcashParameters { pub spend_params: Parameters, From 7fb35bf323e7d7437aca52c90f8adb862e8b737a Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 30 Dec 2021 12:27:08 +1000 Subject: [PATCH 21/29] Derive standard traits on SaplingParameterPaths --- zcash_proofs/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 12268a14e1..155a7a5c29 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -66,6 +66,7 @@ const DOWNLOAD_URL: &str = "https://download.z.cash/downloads"; /// The paths to the Sapling parameter files. #[cfg(feature = "download-params")] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct SaplingParameterPaths { /// The path to the Sapling spend parameter file. pub spend: PathBuf, From 6f537981b91d70541ca20de17efb9f99646040b1 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 7 Jan 2022 07:58:01 +1000 Subject: [PATCH 22/29] Fix error log argument order Also make expected and actual sizes and hashes easier to compare. --- zcash_proofs/src/lib.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 155a7a5c29..ee670fe0ae 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -448,8 +448,9 @@ fn verify_file_size( return Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "{} failed validation: expected {} bytes, \ - actual {} bytes from {:?}", + "{} failed validation:\n\ + expected: {} bytes,\n\ + actual: {} bytes from {:?}", name, expected_bytes, file_size, params_source, ), )); @@ -479,11 +480,14 @@ fn verify_hash( return Err(io::Error::new( read_error.kind(), format!( - "{} failed reading after {} bytes, expected {} bytes from {:?}, error: {:?}", + "{} failed reading:\n\ + expected: {} bytes,\n\ + actual: {} bytes from {:?},\n\ + error: {:?}", name, - params_source, - hash_reader.byte_count(), expected_bytes, + hash_reader.byte_count(), + params_source, read_error, ), )); @@ -495,8 +499,9 @@ fn verify_hash( return Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "{} failed validation: expected: {} hashing {} bytes, \ - actual: {} hashing {} bytes from {:?}", + "{} failed validation:\n\ + expected: {} hashing {} bytes,\n\ + actual: {} hashing {} bytes from {:?}", name, expected_hash, expected_bytes, hash, byte_count, params_source, ), )); From 6d75718076e592a41b6bd6ec916dc15420e4cc3c Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 12 May 2022 09:07:16 +1000 Subject: [PATCH 23/29] Fix clippy::format_push_string --- zcash_proofs/src/hashreader.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zcash_proofs/src/hashreader.rs b/zcash_proofs/src/hashreader.rs index c8663a3d6b..0590e0748d 100644 --- a/zcash_proofs/src/hashreader.rs +++ b/zcash_proofs/src/hashreader.rs @@ -1,5 +1,9 @@ +use std::{ + fmt::Write, + io::{self, Read}, +}; + use blake2b_simd::State; -use std::io::{self, Read}; /// Abstraction over a reader which hashes the data being read. pub struct HashReader { @@ -24,7 +28,7 @@ impl HashReader { let mut s = String::new(); for c in hash.as_bytes().iter() { - s += &format!("{:02x}", c); + write!(&mut s, "{:02x}", c).expect("writing to a string never fails"); } s From 512a785df456ee64abe38448a759d986f282b376 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 29 Jul 2022 09:33:25 +1000 Subject: [PATCH 24/29] Fix hashreader comments Co-authored-by: Kris Nuttycombe --- zcash_proofs/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index ee670fe0ae..f492dd5fb1 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -460,9 +460,9 @@ fn verify_file_size( } /// Check if the Blake2b hash from `hash_reader` matches `expected_hash`, -/// while streaming from `data` into `sink`. +/// while streaming from `hash_reader` into `sink`. /// -/// `hash_reader` can be used to partially read `data`, +/// `hash_reader` can be used to partially read its inner reader's data, /// before verifying the hash using this function. /// /// Returns an error containing `name` and `params_source` on failure. From 9c3317972de262ba544a735571150803d2f51d0b Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 29 Jul 2022 09:35:12 +1000 Subject: [PATCH 25/29] Replace wildcard import --- zcash_proofs/src/downloadreader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_proofs/src/downloadreader.rs b/zcash_proofs/src/downloadreader.rs index af408dae31..b06cc7f99e 100644 --- a/zcash_proofs/src/downloadreader.rs +++ b/zcash_proofs/src/downloadreader.rs @@ -23,7 +23,7 @@ impl From for ResponseLazyReader { impl io::Read for ResponseLazyReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { - use ResponseLazyReader::*; + use ResponseLazyReader::{Request, Response}; // Zero-sized buffer. This should never happen. if buf.is_empty() { From d407b9056c6dcebb16d0cfa505fbe470bdea8d7d Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 29 Jul 2022 11:36:00 +1000 Subject: [PATCH 26/29] Add a missing import --- zcash_proofs/src/downloadreader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zcash_proofs/src/downloadreader.rs b/zcash_proofs/src/downloadreader.rs index b06cc7f99e..e6d9d4321d 100644 --- a/zcash_proofs/src/downloadreader.rs +++ b/zcash_proofs/src/downloadreader.rs @@ -23,7 +23,7 @@ impl From for ResponseLazyReader { impl io::Read for ResponseLazyReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { - use ResponseLazyReader::{Request, Response}; + use ResponseLazyReader::{Complete, Request, Response}; // Zero-sized buffer. This should never happen. if buf.is_empty() { From 1c7cd39f1a21150242e2f6678a334349a4975f6d Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 29 Jul 2022 11:39:49 +1000 Subject: [PATCH 27/29] Require features for the load parameters method --- zcash_proofs/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index f492dd5fb1..c708bd24e6 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -289,6 +289,7 @@ pub struct ZcashParameters { /// Load the specified parameters, checking the sizes and hashes of the files. /// /// Returns the loaded parameters. +#[cfg(any(feature = "local-prover", feature = "download-params"))] pub fn load_parameters( spend_path: &Path, output_path: &Path, From 40d9115cc51bdfe4f55c986c30216ce65e8410ea Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 1 Aug 2022 07:50:10 +1000 Subject: [PATCH 28/29] Fix up conditional compilation and imports --- zcash_proofs/src/hashreader.rs | 2 ++ zcash_proofs/src/lib.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/zcash_proofs/src/hashreader.rs b/zcash_proofs/src/hashreader.rs index 0590e0748d..47fbc648e2 100644 --- a/zcash_proofs/src/hashreader.rs +++ b/zcash_proofs/src/hashreader.rs @@ -1,3 +1,5 @@ +//! Abstraction over a reader which hashes the data being read. + use std::{ fmt::Write, io::{self, Read}, diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index c708bd24e6..1eb68af25b 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -9,11 +9,14 @@ // Temporary until we have addressed all Result cases. #![allow(clippy::result_unit_err)] +use std::{ + fs::File, + io::{self, BufReader}, + path::Path, +}; + use bellman::groth16::{prepare_verifying_key, Parameters, PreparedVerifyingKey, VerifyingKey}; use bls12_381::Bls12; -use std::fs::File; -use std::io::{self, BufReader}; -use std::path::Path; #[cfg(feature = "directories")] use directories::BaseDirs; @@ -22,7 +25,7 @@ use std::path::PathBuf; pub mod circuit; pub mod constants; -mod hashreader; +pub mod hashreader; pub mod sapling; pub mod sprout; @@ -40,15 +43,12 @@ mod downloadreader; // Circuit names /// The sapling spend parameters file name. -#[cfg(any(feature = "local-prover", feature = "download-params"))] pub const SAPLING_SPEND_NAME: &str = "sapling-spend.params"; /// The sapling output parameters file name. -#[cfg(any(feature = "local-prover", feature = "download-params"))] pub const SAPLING_OUTPUT_NAME: &str = "sapling-output.params"; /// The sprout parameters file name. -#[cfg(any(feature = "local-prover", feature = "download-params"))] pub const SPROUT_NAME: &str = "sprout-groth16.params"; // Circuit hashes @@ -66,6 +66,7 @@ const DOWNLOAD_URL: &str = "https://download.z.cash/downloads"; /// The paths to the Sapling parameter files. #[cfg(feature = "download-params")] +#[cfg_attr(docsrs, doc(cfg(feature = "download-params")))] #[derive(Clone, Debug, Eq, PartialEq)] pub struct SaplingParameterPaths { /// The path to the Sapling spend parameter file. @@ -289,7 +290,6 @@ pub struct ZcashParameters { /// Load the specified parameters, checking the sizes and hashes of the files. /// /// Returns the loaded parameters. -#[cfg(any(feature = "local-prover", feature = "download-params"))] pub fn load_parameters( spend_path: &Path, output_path: &Path, From c0f20b347fcef812f4edf61194cb57dbd74bd910 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 1 Aug 2022 07:53:49 +1000 Subject: [PATCH 29/29] Revert unnecessary changes --- zcash_proofs/src/lib.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/zcash_proofs/src/lib.rs b/zcash_proofs/src/lib.rs index 1eb68af25b..2642576d22 100644 --- a/zcash_proofs/src/lib.rs +++ b/zcash_proofs/src/lib.rs @@ -9,14 +9,11 @@ // Temporary until we have addressed all Result cases. #![allow(clippy::result_unit_err)] -use std::{ - fs::File, - io::{self, BufReader}, - path::Path, -}; - use bellman::groth16::{prepare_verifying_key, Parameters, PreparedVerifyingKey, VerifyingKey}; use bls12_381::Bls12; +use std::fs::File; +use std::io::{self, BufReader}; +use std::path::Path; #[cfg(feature = "directories")] use directories::BaseDirs; @@ -25,7 +22,7 @@ use std::path::PathBuf; pub mod circuit; pub mod constants; -pub mod hashreader; +mod hashreader; pub mod sapling; pub mod sprout;