diff --git a/Cargo.lock b/Cargo.lock index b07724e23fc7..b98d343564b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9724,6 +9724,7 @@ name = "zksync_utils" version = "0.1.0" dependencies = [ "anyhow", + "assert_matches", "bigdecimal", "bincode", "futures 0.3.28", diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 36640049e446..a8162de13e9d 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -9,14 +9,14 @@ use zksync_contract_verifier_lib::ContractVerifier; use zksync_core_leftovers::temp_config_store::{load_database_secrets, load_general_config}; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_queued_job_processor::JobProcessor; -use zksync_utils::{wait_for_tasks::ManagedTasks, workspace_dir_or_current_dir}; +use zksync_utils::{env::Workspace, wait_for_tasks::ManagedTasks}; use zksync_vlog::prometheus::PrometheusExporterConfig; async fn update_compiler_versions(connection_pool: &ConnectionPool) { let mut storage = connection_pool.connection().await.unwrap(); let mut transaction = storage.start_transaction().await.unwrap(); - let zksync_home = workspace_dir_or_current_dir(); + let zksync_home = Workspace::locate().core(); let zksolc_path = zksync_home.join("etc/zksolc-bin/"); let zksolc_versions: Vec = std::fs::read_dir(zksolc_path) diff --git a/core/bin/system-constants-generator/src/main.rs b/core/bin/system-constants-generator/src/main.rs index 7ada47302248..cc2e031106b8 100644 --- a/core/bin/system-constants-generator/src/main.rs +++ b/core/bin/system-constants-generator/src/main.rs @@ -17,7 +17,7 @@ use zksync_types::{ IntrinsicSystemGasConstants, ProtocolVersionId, GUARANTEED_PUBDATA_IN_TX, L1_GAS_PER_PUBDATA_BYTE, MAX_NEW_FACTORY_DEPS, REQUIRED_L1_TO_L2_GAS_PER_PUBDATA_BYTE, }; -use zksync_utils::workspace_dir_or_current_dir; +use zksync_utils::env::Workspace; // For configs we will use the default value of `800_000` to represent the rough amount of L1 gas // needed to cover the batch expenses. @@ -210,7 +210,7 @@ fn generate_rust_fee_constants(intrinsic_gas_constants: &IntrinsicSystemGasConst } fn save_file(path_in_repo: &str, content: String) { - let zksync_home = workspace_dir_or_current_dir(); + let zksync_home = Workspace::locate().core(); let fee_constants_path = zksync_home.join(path_in_repo); fs::write(fee_constants_path, content) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 82751d4c9754..c8d9b89d834c 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -1,6 +1,6 @@ use std::{ collections::HashMap, - path::Path, + path::{Path, PathBuf}, time::{Duration, Instant}, }; @@ -20,7 +20,7 @@ use zksync_types::{ }, Address, }; -use zksync_utils::workspace_dir_or_current_dir; +use zksync_utils::env::Workspace; use crate::{ error::ContractVerifierError, @@ -38,8 +38,8 @@ lazy_static! { static ref DEPLOYER_CONTRACT: Contract = zksync_contracts::deployer_contract(); } -fn home_path() -> &'static Path { - workspace_dir_or_current_dir() +fn home_path() -> PathBuf { + Workspace::locate().core() } #[derive(Debug)] diff --git a/core/lib/contracts/src/lib.rs b/core/lib/contracts/src/lib.rs index f10e557a642d..f57649c9d695 100644 --- a/core/lib/contracts/src/lib.rs +++ b/core/lib/contracts/src/lib.rs @@ -16,7 +16,7 @@ use ethabi::{ }; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; -use zksync_utils::{bytecode::hash_bytecode, bytes_to_be_words, workspace_dir_or_current_dir}; +use zksync_utils::{bytecode::hash_bytecode, bytes_to_be_words, env::Workspace}; pub mod test_contracts; @@ -64,8 +64,8 @@ const LOADNEXT_CONTRACT_FILE: &str = const LOADNEXT_SIMPLE_CONTRACT_FILE: &str = "etc/contracts-test-data/artifacts-zk/contracts/loadnext/loadnext_contract.sol/Foo.json"; -fn home_path() -> &'static Path { - workspace_dir_or_current_dir() +fn home_path() -> PathBuf { + Workspace::locate().core() } fn read_file_to_json_value(path: impl AsRef + std::fmt::Debug) -> serde_json::Value { diff --git a/core/lib/utils/Cargo.toml b/core/lib/utils/Cargo.toml index 593952f16aca..b87b2ad98964 100644 --- a/core/lib/utils/Cargo.toml +++ b/core/lib/utils/Cargo.toml @@ -32,3 +32,4 @@ once_cell.workspace = true rand.workspace = true tokio = { workspace = true, features = ["macros", "rt"] } bincode.workspace = true +assert_matches.workspace = true diff --git a/core/lib/utils/src/env.rs b/core/lib/utils/src/env.rs index 0eddc6c2cd64..5ae07caf1486 100644 --- a/core/lib/utils/src/env.rs +++ b/core/lib/utils/src/env.rs @@ -8,6 +8,87 @@ use once_cell::sync::OnceCell; static WORKSPACE: OnceCell> = OnceCell::new(); +/// Represents Cargo workspaces available in the repository. +#[derive(Debug, Clone, Copy)] +pub enum Workspace<'a> { + /// Workspace was not found. + /// Assumes that the code is running in a binary. + /// Will use the current directory as a fallback. + None, + /// Root folder. + Core(&'a Path), + /// `prover` folder. + Prover(&'a Path), + /// `toolbox` folder. + Toolbox(&'a Path), +} + +impl Workspace<'static> { + /// Find the location of the current workspace, if this code works in workspace + /// then it will return the correct folder if, it's binary e.g. in docker container + /// you have to use fallback to another directory + /// The code has been inspired by `insta` + /// `https://github.com/mitsuhiko/insta/blob/master/insta/src/env.rs` + pub fn locate() -> Self { + // Since `locate_workspace_inner()` should be deterministic, it makes little sense to call + // `OnceCell::get_or_try_init()` here; the repeated calls are just as unlikely to succeed as the initial call. + // Instead, we store `None` in the `OnceCell` if initialization failed. + let path: Option<&'static Path> = WORKSPACE + .get_or_init(|| { + let result = locate_workspace_inner(); + // If the workspace is not found, we store `None` in the `OnceCell`. + // It doesn't make sense to log it, since in most production cases the workspace + // is not present. + result.ok() + }) + .as_deref(); + path.map_or(Self::None, Self::from) + } +} + +impl<'a> Workspace<'a> { + const PROVER_DIRECTORY_NAME: &'static str = "prover"; + const TOOLBOX_DIRECTORY_NAME: &'static str = "zk_toolbox"; + + /// Returns the path of the core workspace. + /// For `Workspace::None`, considers the current directory to represent core workspace. + pub fn core(self) -> PathBuf { + match self { + Self::None => PathBuf::from("."), + Self::Core(path) => path.into(), + Self::Prover(path) | Self::Toolbox(path) => path.parent().unwrap().into(), + } + } + + /// Returns the path of the `prover` workspace. + pub fn prover(self) -> PathBuf { + match self { + Self::Prover(path) => path.into(), + _ => self.core().join(Self::PROVER_DIRECTORY_NAME), + } + } + + /// Returns the path of the `zk_toolbox`` workspace. + pub fn toolbox(self) -> PathBuf { + match self { + Self::Toolbox(path) => path.into(), + _ => self.core().join(Self::TOOLBOX_DIRECTORY_NAME), + } + } +} + +impl<'a> From<&'a Path> for Workspace<'a> { + fn from(path: &'a Path) -> Self { + if path.ends_with(Self::PROVER_DIRECTORY_NAME) { + Self::Prover(path) + } else if path.ends_with(Self::TOOLBOX_DIRECTORY_NAME) { + Self::Toolbox(path) + } else { + Self::Core(path) + } + } +} + fn locate_workspace_inner() -> anyhow::Result { let output = std::process::Command::new( std::env::var("CARGO") @@ -40,31 +121,86 @@ fn locate_workspace_inner() -> anyhow::Result { .to_path_buf()) } -/// Find the location of the current workspace, if this code works in workspace -/// then it will return the correct folder if, it's binary e.g. in docker container -/// you have to use fallback to another directory -/// The code has been inspired by `insta` -/// `https://github.com/mitsuhiko/insta/blob/master/insta/src/env.rs` -pub fn locate_workspace() -> Option<&'static Path> { - // Since `locate_workspace_inner()` should be deterministic, it makes little sense to call - // `OnceCell::get_or_try_init()` here; the repeated calls are just as unlikely to succeed as the initial call. - // Instead, we store `None` in the `OnceCell` if initialization failed. - WORKSPACE - .get_or_init(|| { - let result = locate_workspace_inner(); - if result.is_err() { - // `get_or_init()` is guaranteed to call the provided closure once per `OnceCell`; - // i.e., we won't spam logs here. - tracing::info!( - "locate_workspace() failed. You are using an already compiled version" - ); - } - result.ok() - }) - .as_deref() -} +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + + use super::*; + + /// Will reset the pwd on drop. + /// This is needed to make sure that even if the test fails, the env + /// for other tests is left intact. + struct PwdProtector(PathBuf); + + impl PwdProtector { + fn new() -> Self { + let pwd = std::env::current_dir().unwrap(); + Self(pwd) + } + } + + impl Drop for PwdProtector { + fn drop(&mut self) { + std::env::set_current_dir(self.0.clone()).unwrap(); + } + } + + #[test] + fn test_workspace_locate() { + let _pwd_protector = PwdProtector::new(); + + // Core. + + let workspace = Workspace::locate(); + assert_matches!(workspace, Workspace::Core(_)); + let core_path = workspace.core(); + // Check if prover and toolbox directories exist. + assert!(workspace.prover().exists()); + assert_matches!( + Workspace::from(workspace.prover().as_path()), + Workspace::Prover(_) + ); + assert!(workspace.toolbox().exists()); + assert_matches!( + Workspace::from(workspace.toolbox().as_path()), + Workspace::Toolbox(_) + ); + + // Prover. + + // We use `cargo-nextest` for running tests, which runs each test in parallel, + // so we can safely alter the global env, assuming that we will restore it after + // the test. + std::env::set_current_dir(workspace.prover()).unwrap(); + let workspace_path = locate_workspace_inner().unwrap(); + let workspace = Workspace::from(workspace_path.as_path()); + assert_matches!(workspace, Workspace::Prover(_)); + let prover_path = workspace.prover(); + assert_eq!(workspace.core(), core_path); + assert_matches!( + Workspace::from(workspace.core().as_path()), + Workspace::Core(_) + ); + assert!(workspace.toolbox().exists()); + assert_matches!( + Workspace::from(workspace.toolbox().as_path()), + Workspace::Toolbox(_) + ); -/// Returns [`locate_workspace()`] output with the "." fallback. -pub fn workspace_dir_or_current_dir() -> &'static Path { - locate_workspace().unwrap_or_else(|| Path::new(".")) + // Toolbox. + std::env::set_current_dir(workspace.toolbox()).unwrap(); + let workspace_path = locate_workspace_inner().unwrap(); + let workspace = Workspace::from(workspace_path.as_path()); + assert_matches!(workspace, Workspace::Toolbox(_)); + assert_eq!(workspace.core(), core_path); + assert_matches!( + Workspace::from(workspace.core().as_path()), + Workspace::Core(_) + ); + assert_eq!(workspace.prover(), prover_path); + assert_matches!( + Workspace::from(workspace.prover().as_path()), + Workspace::Prover(_) + ); + } } diff --git a/core/lib/utils/src/lib.rs b/core/lib/utils/src/lib.rs index 7f9304e3110c..92a1d7a0c470 100644 --- a/core/lib/utils/src/lib.rs +++ b/core/lib/utils/src/lib.rs @@ -2,7 +2,7 @@ pub mod bytecode; mod convert; -mod env; +pub mod env; pub mod http_with_retries; pub mod misc; pub mod panic_extractor; @@ -10,4 +10,4 @@ mod serde_wrappers; pub mod time; pub mod wait_for_tasks; -pub use self::{convert::*, env::*, misc::*, serde_wrappers::*}; +pub use self::{convert::*, misc::*, serde_wrappers::*}; diff --git a/core/tests/loadnext/src/config.rs b/core/tests/loadnext/src/config.rs index a9648edb00ae..ab578ecfdc6b 100644 --- a/core/tests/loadnext/src/config.rs +++ b/core/tests/loadnext/src/config.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use tokio::sync::Semaphore; use zksync_contracts::test_contracts::LoadnextContractExecutionParams; use zksync_types::{network::Network, Address, L2ChainId, H160}; -use zksync_utils::workspace_dir_or_current_dir; +use zksync_utils::env::Workspace; use crate::fs_utils::read_tokens; @@ -190,7 +190,7 @@ fn default_main_token() -> H160 { } fn default_test_contracts_path() -> PathBuf { - let test_contracts_path = workspace_dir_or_current_dir().join("etc/contracts-test-data"); + let test_contracts_path = Workspace::locate().core().join("etc/contracts-test-data"); tracing::info!("Test contracts path: {}", test_contracts_path.display()); test_contracts_path } diff --git a/core/tests/loadnext/src/fs_utils.rs b/core/tests/loadnext/src/fs_utils.rs index 8af9df8afee7..c4472a00531c 100644 --- a/core/tests/loadnext/src/fs_utils.rs +++ b/core/tests/loadnext/src/fs_utils.rs @@ -5,7 +5,7 @@ use std::{fs::File, io::BufReader, path::Path}; use serde::Deserialize; use zksync_types::{ethabi::Contract, network::Network, Address}; -use zksync_utils::workspace_dir_or_current_dir; +use zksync_utils::env::Workspace; /// A token stored in `etc/tokens/{network}.json` files. #[derive(Debug, Deserialize)] @@ -27,7 +27,7 @@ pub struct TestContract { } pub fn read_tokens(network: Network) -> anyhow::Result> { - let home = workspace_dir_or_current_dir(); + let home = Workspace::locate().core(); let path = home.join(format!("etc/tokens/{network}.json")); let file = File::open(path)?; let reader = BufReader::new(file); diff --git a/prover/Cargo.lock b/prover/Cargo.lock index e77bb4f488bb..21e2ea8b21de 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -8189,6 +8189,7 @@ dependencies = [ "zksync_prover_fri_types", "zksync_prover_keystore", "zksync_types", + "zksync_utils", "zksync_vlog", ] diff --git a/prover/crates/bin/prover_cli/src/config/mod.rs b/prover/crates/bin/prover_cli/src/config/mod.rs index 3d99f2be3b2c..b3df2e7d2c56 100644 --- a/prover/crates/bin/prover_cli/src/config/mod.rs +++ b/prover/crates/bin/prover_cli/src/config/mod.rs @@ -1,12 +1,12 @@ use std::{io::Write, path::PathBuf}; -use crate::helper::core_workspace_dir_or_current_dir; +use zksync_utils::env::Workspace; pub fn get_envfile() -> anyhow::Result { if let Ok(envfile) = std::env::var("PLI__CONFIG") { return Ok(envfile.into()); } - Ok(core_workspace_dir_or_current_dir().join("etc/pliconfig")) + Ok(Workspace::locate().core().join("etc/pliconfig")) } pub fn load_envfile(path: impl AsRef) -> anyhow::Result<()> { diff --git a/prover/crates/bin/prover_cli/src/helper.rs b/prover/crates/bin/prover_cli/src/helper.rs index 352a789baed7..7fe0c990e4e0 100644 --- a/prover/crates/bin/prover_cli/src/helper.rs +++ b/prover/crates/bin/prover_cli/src/helper.rs @@ -1,10 +1,7 @@ -use std::{ - fs::File, - path::{Path, PathBuf}, -}; +use std::{fs::File, path::PathBuf}; use zksync_types::ethabi::Contract; -use zksync_utils::locate_workspace; +use zksync_utils::env::Workspace; const ZKSYNC_HYPERCHAIN_CONTRACT_FILE: &str = "contracts/l1-contracts/artifacts/contracts/state-transition/chain-interfaces/IZkSyncHyperchain.sol/IZkSyncHyperchain.json"; @@ -27,8 +24,7 @@ fn read_file_to_json_value(path: &PathBuf) -> serde_json::Value { } fn load_contract_if_present(path: &str) -> Contract { - let home = core_workspace_dir_or_current_dir(); - let path = Path::new(&home).join(path); + let path = Workspace::locate().core().join(path); path.exists() .then(|| { serde_json::from_value(read_file_to_json_value(&path)["abi"].take()).unwrap_or_else( @@ -39,9 +35,3 @@ fn load_contract_if_present(path: &str) -> Contract { panic!("Failed to load contract from {:?}", path); }) } - -pub fn core_workspace_dir_or_current_dir() -> PathBuf { - locate_workspace() - .map(|a| a.join("..")) - .unwrap_or_else(|| PathBuf::from(".")) -} diff --git a/prover/crates/bin/vk_setup_data_generator_server_fri/Cargo.toml b/prover/crates/bin/vk_setup_data_generator_server_fri/Cargo.toml index 7c17e845450c..4830f2277a79 100644 --- a/prover/crates/bin/vk_setup_data_generator_server_fri/Cargo.toml +++ b/prover/crates/bin/vk_setup_data_generator_server_fri/Cargo.toml @@ -20,6 +20,7 @@ zksync_vlog.workspace = true zksync_types.workspace = true zksync_prover_fri_types.workspace = true zksync_prover_keystore.workspace = true +zksync_utils.workspace = true zkevm_test_harness.workspace = true circuit_definitions = { workspace = true, features = ["log_tracing"] } diff --git a/prover/crates/bin/vk_setup_data_generator_server_fri/src/vk_commitment_helper.rs b/prover/crates/bin/vk_setup_data_generator_server_fri/src/vk_commitment_helper.rs index 02cbe6e0c4de..2753799dc722 100644 --- a/prover/crates/bin/vk_setup_data_generator_server_fri/src/vk_commitment_helper.rs +++ b/prover/crates/bin/vk_setup_data_generator_server_fri/src/vk_commitment_helper.rs @@ -2,7 +2,7 @@ use std::{fs, path::PathBuf}; use anyhow::Context as _; use toml_edit::{Document, Item, Value}; -use zksync_prover_keystore::utils::core_workspace_dir_or_current_dir; +use zksync_utils::env::Workspace; pub fn get_toml_formatted_value(string_value: String) -> Item { let mut value = Value::from(string_value); @@ -23,5 +23,7 @@ pub fn read_contract_toml() -> anyhow::Result { } pub fn get_contract_toml_path() -> PathBuf { - core_workspace_dir_or_current_dir().join("etc/env/base/contracts.toml") + Workspace::locate() + .core() + .join("etc/env/base/contracts.toml") } diff --git a/prover/crates/lib/keystore/src/keystore.rs b/prover/crates/lib/keystore/src/keystore.rs index ff14387bfda7..28ce989287cc 100644 --- a/prover/crates/lib/keystore/src/keystore.rs +++ b/prover/crates/lib/keystore/src/keystore.rs @@ -18,10 +18,11 @@ use serde::{Deserialize, Serialize}; use zkevm_test_harness::data_source::{in_memory_data_source::InMemoryDataSource, SetupDataSource}; use zksync_basic_types::basic_fri_types::AggregationRound; use zksync_prover_fri_types::ProverServiceDataKey; +use zksync_utils::env::Workspace; #[cfg(feature = "gpu")] use crate::GoldilocksGpuProverSetupData; -use crate::{utils::core_workspace_dir_or_current_dir, GoldilocksProverSetupData, VkCommitments}; +use crate::{GoldilocksProverSetupData, VkCommitments}; pub enum ProverServiceDataType { VerificationKey, @@ -42,31 +43,6 @@ pub struct Keystore { setup_data_path: PathBuf, } -fn get_base_path() -> PathBuf { - // This will return the path to the _core_ workspace locally, - // otherwise (e.g. in Docker) it will return `.` (which is usually equivalent to `/`). - // - // Note: at the moment of writing this function, it locates the prover workspace, and uses - // `..` to get to the core workspace, so the path returned is something like: - // `/path/to/workspace/zksync-era/prover/..` (or `.` for binaries). - let path = core_workspace_dir_or_current_dir(); - - // Check if we're in the folder equivalent to the core workspace root. - // Path we're actually checking is: - // `/path/to/workspace/zksync-era/prover/../prover/data/keys` - let new_path = path.join("prover/data/keys"); - if new_path.exists() { - return new_path; - } - - let mut components = path.components(); - // This removes the last component of `path`, so: - // for local workspace, we're removing `..` and putting ourselves back to the prover workspace. - // for binaries, we're removing `.` and getting the empty path. - components.next_back().unwrap(); - components.as_path().join("prover/data/keys") -} - impl Keystore { /// Base-dir is the location of smaller keys (like verification keys and finalization hints). /// Setup data path is used for the large setup keys. @@ -79,8 +55,33 @@ impl Keystore { /// Uses automatic detection of the base path, and assumes that setup keys /// are stored in the same directory. + /// + /// The "base" path is considered to be equivalent to the `prover/data/keys` + /// directory in the repository. pub fn locate() -> Self { - let base_path = get_base_path(); + // There might be several cases: + // - We're running from the prover workspace. + // - We're running from the core workspace. + // - We're running the binary from the docker. + let data_dir_path = match Workspace::locate() { + Workspace::None => { + // We're running a binary, likely in a docker. + // Keys can be in one of a few paths. + // We want to be very conservative here, and checking + // more locations than we likely need to not accidentally + // break something. + let paths = ["./prover/data", "./data", "/prover/data", "/data"]; + paths.iter().map(PathBuf::from).find(|path| path.exists()).unwrap_or_else(|| { + panic!("Failed to locate the prover data directory. Locations checked: {paths:?}") + }) + } + ws => { + // If we're running in the Cargo workspace, the data *must* be in `prover/data`. + ws.prover().join("data") + } + }; + let base_path = data_dir_path.join("keys"); + Self { basedir: base_path.clone(), setup_data_path: base_path, diff --git a/prover/crates/lib/keystore/src/utils.rs b/prover/crates/lib/keystore/src/utils.rs index 5cebf7aef77a..d9bb3b47dbb0 100644 --- a/prover/crates/lib/keystore/src/utils.rs +++ b/prover/crates/lib/keystore/src/utils.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - use anyhow::Context as _; use circuit_definitions::{ circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuit, @@ -22,7 +20,6 @@ use zksync_prover_fri_types::circuit_definitions::{ scheduler::aux::BaseLayerCircuitType, }, }; -use zksync_utils::locate_workspace; use crate::keystore::Keystore; @@ -115,24 +112,16 @@ pub fn calculate_snark_vk_hash(keystore: &Keystore) -> anyhow::Result { Ok(H256::from_slice(&computed_vk_hash)) } -/// Returns workspace of the core component, we assume that prover is one folder deeper. -/// Or fallback to current dir -pub fn core_workspace_dir_or_current_dir() -> PathBuf { - locate_workspace() - .map(|a| a.join("..")) - .unwrap_or_else(|| PathBuf::from(".")) -} - #[cfg(test)] mod tests { - use std::{path::PathBuf, str::FromStr}; + use std::str::FromStr; + use zksync_utils::env::Workspace; use super::*; #[test] fn test_keyhash_generation() { - let mut path_to_input = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); - path_to_input.push("../../../data/historical_data"); + let path_to_input = Workspace::locate().prover().join("data/historical_data"); for entry in std::fs::read_dir(path_to_input.clone()).unwrap().flatten() { if entry.metadata().unwrap().is_dir() {