From 2f0614706d91349e443d93f0449b9d52f55a3d71 Mon Sep 17 00:00:00 2001 From: Shahar Kaminsky Date: Tue, 9 Jul 2024 20:12:19 +0300 Subject: [PATCH] feat: Minimal External API Fetcher (#2383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ For chains running with a custom base token, use live prices in creating ETH<->BaseToken conversion ratios. This PR - adds a first `PriceAPIClientResource` with a `PriceAPIClient` CoinGecko client. - Uses this client in the `BaseTokenRatioPersister` to fetch new prices and create new ratios Not included in this PR and will be follow-up work: - Redundancy in the number of external price feeds used - Different "strategies" for how true price converged from multiple price feeds ## Why ❔ For the base token flow, we need to be be able to fetch live prices of BaseToken & ETH from which to create the conversion ratios. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. --------- Co-authored-by: dimazhornyk Co-authored-by: Igor Aleksanov --- .github/workflows/ci-core-reusable.yml | 4 +- Cargo.lock | 85 ++++++++----- Cargo.toml | 1 + core/bin/zksync_server/src/main.rs | 10 +- core/bin/zksync_server/src/node_builder.rs | 38 +++++- core/lib/config/Cargo.toml | 1 + .../src/configs/external_price_api_client.rs | 27 +++++ core/lib/config/src/configs/general.rs | 8 +- core/lib/config/src/configs/mod.rs | 2 + .../src/external_price_api_client.rs | 48 ++++++++ core/lib/env_config/src/lib.rs | 1 + core/lib/external_price_api/Cargo.toml | 8 ++ .../external_price_api/src/coingecko_api.rs | 112 ++++++++++++++++++ .../src/forced_price_client.rs | 62 ++++++++++ core/lib/external_price_api/src/lib.rs | 25 +++- core/lib/external_price_api/src/utils.rs | 15 +++ .../src/external_price_api_client.rs | 32 +++++ core/lib/protobuf_config/src/general.rs | 6 + core/lib/protobuf_config/src/lib.rs | 1 + .../config/external_price_api_client.proto | 12 ++ .../src/proto/config/general.proto | 2 + core/lib/types/src/base_token_ratio.rs | 14 ++- .../src/temp_config_store/mod.rs | 10 +- core/node/base_token_adjuster/Cargo.toml | 1 + .../src/base_token_ratio_persister.rs | 95 ++++++++++----- core/node/node_framework/Cargo.toml | 1 + .../base_token_ratio_persister.rs | 32 ++++- .../base_token_ratio_provider.rs | 0 .../layers/base_token/coingecko_client.rs | 55 +++++++++ .../layers/base_token/forced_price_client.rs | 52 ++++++++ .../implementations/layers/base_token/mod.rs | 5 + .../no_op_external_price_api_client.rs | 45 +++++++ .../src/implementations/layers/mod.rs | 3 +- .../src/implementations/resources/mod.rs | 1 + .../resources/price_api_client.rs | 27 +++++ etc/env/base/external_price_api.toml | 8 ++ etc/env/base/rust.toml | 1 + etc/env/file_based/general.yaml | 5 +- prover/Cargo.lock | 1 + prover/config/src/lib.rs | 9 +- zk_toolbox/Cargo.lock | 1 + 41 files changed, 775 insertions(+), 91 deletions(-) create mode 100644 core/lib/config/src/configs/external_price_api_client.rs create mode 100644 core/lib/env_config/src/external_price_api_client.rs create mode 100644 core/lib/external_price_api/src/coingecko_api.rs create mode 100644 core/lib/external_price_api/src/forced_price_client.rs create mode 100644 core/lib/external_price_api/src/utils.rs create mode 100644 core/lib/protobuf_config/src/external_price_api_client.rs create mode 100644 core/lib/protobuf_config/src/proto/config/external_price_api_client.proto rename core/node/node_framework/src/implementations/layers/{ => base_token}/base_token_ratio_persister.rs (63%) rename core/node/node_framework/src/implementations/layers/{ => base_token}/base_token_ratio_provider.rs (100%) create mode 100644 core/node/node_framework/src/implementations/layers/base_token/coingecko_client.rs create mode 100644 core/node/node_framework/src/implementations/layers/base_token/forced_price_client.rs create mode 100644 core/node/node_framework/src/implementations/layers/base_token/mod.rs create mode 100644 core/node/node_framework/src/implementations/layers/base_token/no_op_external_price_api_client.rs create mode 100644 core/node/node_framework/src/implementations/resources/price_api_client.rs create mode 100644 etc/env/base/external_price_api.toml diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 504f7761bb8e..93aa1bb1658b 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -135,7 +135,7 @@ jobs: base_token: ["Eth", "Custom"] deployment_mode: ["Rollup", "Validium"] env: - SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher,base_token_ratio_persister${{ matrix.consensus && ',consensus' || '' }}" + SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}" runs-on: [matterlabs-ci-runner] steps: @@ -309,7 +309,7 @@ jobs: runs-on: [matterlabs-ci-runner] env: - SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher,base_token_ratio_persister${{ matrix.consensus && ',consensus' || '' }}" + SERVER_COMPONENTS: "api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,da_dispatcher${{ matrix.consensus && ',consensus' || '' }}${{ matrix.base_token == 'Custom' && ',base_token_ratio_persister' || '' }}" EXT_NODE_FLAGS: "${{ matrix.consensus && '-- --enable-consensus' || '' }}" steps: diff --git a/Cargo.lock b/Cargo.lock index 750f64f794af..dcb41a6fa931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,7 +553,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" dependencies = [ - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -2127,7 +2127,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b96fbccd88dbb1fac4ee4a07c2fcc4ca719a74ffbd9d2b9d41d8c8eb073d8b20" dependencies = [ - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-integer", "num-traits", "proc-macro2 1.0.69", @@ -2235,6 +2235,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "franklin-crypto" version = "0.1.0" @@ -2253,7 +2263,7 @@ dependencies = [ "indexmap 1.9.3", "itertools 0.10.5", "lazy_static", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-derive", "num-integer", "num-traits", @@ -3157,6 +3167,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.0" @@ -3881,11 +3900,11 @@ dependencies = [ [[package]] name = "num" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-complex", "num-integer", "num-iter", @@ -3906,11 +3925,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", "serde", @@ -3935,9 +3953,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", "serde", @@ -3956,19 +3974,18 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -3987,12 +4004,11 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-integer", "num-traits", "serde", @@ -4000,9 +4016,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -4743,7 +4759,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.10.5", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -4777,7 +4793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.11.0", "proc-macro2 1.0.69", "quote 1.0.33", "syn 2.0.38", @@ -5575,7 +5591,7 @@ dependencies = [ "core-foundation", "core-foundation-sys", "libc", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "security-framework-sys", ] @@ -5958,7 +5974,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-traits", "thiserror", "time", @@ -6284,7 +6300,7 @@ dependencies = [ "log", "md-5", "memchr", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "once_cell", "rand 0.8.5", "rust_decimal", @@ -7996,6 +8012,7 @@ dependencies = [ "tracing", "zksync_config", "zksync_dal", + "zksync_external_price_api", "zksync_types", ] @@ -8116,6 +8133,7 @@ dependencies = [ "rand 0.8.5", "secrecy", "serde", + "url", "zksync_basic_types", "zksync_concurrency", "zksync_consensus_utils", @@ -8157,7 +8175,7 @@ dependencies = [ "ff_ce", "hex", "k256 0.13.3", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "num-traits", "pairing_ce", "rand 0.4.6", @@ -8232,7 +8250,7 @@ dependencies = [ "anyhow", "bit-vec", "hex", - "num-bigint 0.4.4", + "num-bigint 0.4.6", "prost 0.12.1", "rand 0.8.5", "serde", @@ -8673,6 +8691,14 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "bigdecimal", + "chrono", + "fraction", + "rand 0.8.5", + "reqwest 0.12.5", + "serde", + "tokio", + "url", "zksync_config", "zksync_types", ] @@ -9024,6 +9050,7 @@ dependencies = [ "zksync_eth_client", "zksync_eth_sender", "zksync_eth_watch", + "zksync_external_price_api", "zksync_health_check", "zksync_house_keeper", "zksync_metadata_calculator", diff --git a/Cargo.toml b/Cargo.toml index 443f85493865..8b1be4471707 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,6 +179,7 @@ tracing-subscriber = "0.3" tracing-opentelemetry = "0.21.0" url = "2" web3 = "0.19.0" +fraction = "0.15.3" # Proc-macro syn = "2.0" diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index 4612a737bacc..b589d04aed66 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -12,9 +12,10 @@ use zksync_config::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, BasicWitnessInputProducerConfig, ContractsConfig, DatabaseSecrets, - FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, - FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, L1Secrets, ObservabilityConfig, - PrometheusConfig, ProofDataHandlerConfig, ProtectiveReadsWriterConfig, Secrets, + ExternalPriceApiClientConfig, FriProofCompressorConfig, FriProverConfig, + FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, + L1Secrets, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, + ProtectiveReadsWriterConfig, Secrets, }, ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, GenesisConfig, ObjectStoreConfig, PostgresConfig, @@ -43,7 +44,7 @@ struct Cli { /// Comma-separated list of components to launch. #[arg( long, - default_value = "api,tree,eth,state_keeper,housekeeper,tee_verifier_input_producer,commitment_generator,da_dispatcher,base_token_ratio_persister" + default_value = "api,tree,eth,state_keeper,housekeeper,tee_verifier_input_producer,commitment_generator,da_dispatcher" )] components: ComponentsToRun, /// Path to the yaml config. If set, it will be used instead of env vars. @@ -230,5 +231,6 @@ fn load_env_config() -> anyhow::Result { commitment_generator: None, pruning: None, snapshot_recovery: None, + external_price_api_client_config: ExternalPriceApiClientConfig::from_env().ok(), }) } diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 46cafe227f9a..f8173579b57e 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -21,8 +21,12 @@ use zksync_node_api_server::{ }; use zksync_node_framework::{ implementations::layers::{ - base_token_ratio_persister::BaseTokenRatioPersisterLayer, - base_token_ratio_provider::BaseTokenRatioProviderLayer, + base_token::{ + base_token_ratio_persister::BaseTokenRatioPersisterLayer, + base_token_ratio_provider::BaseTokenRatioProviderLayer, + coingecko_client::CoingeckoClientLayer, forced_price_client::ForcedPriceClientLayer, + no_op_external_price_api_client::NoOpExternalPriceApiClientLayer, + }, circuit_breaker_checker::CircuitBreakerCheckerLayer, commitment_generator::CommitmentGeneratorLayer, consensus::MainNodeConsensusLayer, @@ -516,6 +520,29 @@ impl MainNodeBuilder { Ok(self) } + fn add_external_api_client_layer(mut self) -> anyhow::Result { + let config = try_load_config!(self.configs.external_price_api_client_config); + match config.source.as_str() { + CoingeckoClientLayer::CLIENT_NAME => { + self.node.add_layer(CoingeckoClientLayer::new(config)); + } + NoOpExternalPriceApiClientLayer::CLIENT_NAME => { + self.node.add_layer(NoOpExternalPriceApiClientLayer); + } + ForcedPriceClientLayer::CLIENT_NAME => { + self.node.add_layer(ForcedPriceClientLayer::new(config)); + } + _ => { + anyhow::bail!( + "Unknown external price API client source: {}", + config.source + ); + } + } + + Ok(self) + } + fn add_vm_runner_bwip_layer(mut self) -> anyhow::Result { let basic_witness_input_producer_config = try_load_config!(self.configs.basic_witness_input_producer_config); @@ -529,8 +556,9 @@ impl MainNodeBuilder { fn add_base_token_ratio_persister_layer(mut self) -> anyhow::Result { let config = try_load_config!(self.configs.base_token_adjuster); + let contracts_config = self.contracts_config.clone(); self.node - .add_layer(BaseTokenRatioPersisterLayer::new(config)); + .add_layer(BaseTokenRatioPersisterLayer::new(config, contracts_config)); Ok(self) } @@ -669,7 +697,9 @@ impl MainNodeBuilder { self = self.add_vm_runner_protective_reads_layer()?; } Component::BaseTokenRatioPersister => { - self = self.add_base_token_ratio_persister_layer()?; + self = self + .add_external_api_client_layer()? + .add_base_token_ratio_persister_layer()?; } Component::VmRunnerBwip => { self = self.add_vm_runner_bwip_layer()?; diff --git a/core/lib/config/Cargo.toml b/core/lib/config/Cargo.toml index 2e1da7d0f3a2..551b97cc0b9b 100644 --- a/core/lib/config/Cargo.toml +++ b/core/lib/config/Cargo.toml @@ -15,6 +15,7 @@ zksync_crypto_primitives.workspace = true zksync_consensus_utils.workspace = true zksync_concurrency.workspace = true +url.workspace = true anyhow.workspace = true rand.workspace = true secrecy.workspace = true diff --git a/core/lib/config/src/configs/external_price_api_client.rs b/core/lib/config/src/configs/external_price_api_client.rs new file mode 100644 index 000000000000..06282eb8bebd --- /dev/null +++ b/core/lib/config/src/configs/external_price_api_client.rs @@ -0,0 +1,27 @@ +use std::time::Duration; + +use serde::Deserialize; + +pub const DEFAULT_TIMEOUT_MS: u64 = 10_000; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ExternalPriceApiClientConfig { + pub source: String, + pub base_url: Option, + pub api_key: Option, + #[serde(default = "ExternalPriceApiClientConfig::default_timeout")] + pub client_timeout_ms: u64, + /// Forced conversion ratio. Only used with the ForcedPriceClient. + pub forced_numerator: Option, + pub forced_denominator: Option, +} + +impl ExternalPriceApiClientConfig { + fn default_timeout() -> u64 { + DEFAULT_TIMEOUT_MS + } + + pub fn client_timeout(&self) -> Duration { + Duration::from_millis(self.client_timeout_ms) + } +} diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs index 9dbda3f845ee..e80538b2a4b9 100644 --- a/core/lib/config/src/configs/general.rs +++ b/core/lib/config/src/configs/general.rs @@ -8,9 +8,10 @@ use crate::{ pruning::PruningConfig, snapshot_recovery::SnapshotRecoveryConfig, vm_runner::{BasicWitnessInputProducerConfig, ProtectiveReadsWriterConfig}, - CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, - FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, - ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, + CommitmentGeneratorConfig, ExternalPriceApiClientConfig, FriProofCompressorConfig, + FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, + FriWitnessVectorGeneratorConfig, ObservabilityConfig, PrometheusConfig, + ProofDataHandlerConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -46,4 +47,5 @@ pub struct GeneralConfig { pub pruning: Option, pub core_object_store: Option, pub base_token_adjuster: Option, + pub external_price_api_client_config: Option, } diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index f66b6f897125..0da6f986f353 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -10,6 +10,7 @@ pub use self::{ eth_sender::{EthConfig, GasAdjusterConfig}, eth_watch::EthWatchConfig, experimental::ExperimentalDBConfig, + external_price_api_client::ExternalPriceApiClientConfig, fri_proof_compressor::FriProofCompressorConfig, fri_prover::FriProverConfig, fri_prover_gateway::FriProverGatewayConfig, @@ -41,6 +42,7 @@ pub mod en_config; pub mod eth_sender; pub mod eth_watch; mod experimental; +pub mod external_price_api_client; pub mod fri_proof_compressor; pub mod fri_prover; pub mod fri_prover_gateway; diff --git a/core/lib/env_config/src/external_price_api_client.rs b/core/lib/env_config/src/external_price_api_client.rs new file mode 100644 index 000000000000..7ec3782dc6b4 --- /dev/null +++ b/core/lib/env_config/src/external_price_api_client.rs @@ -0,0 +1,48 @@ +use zksync_config::configs::ExternalPriceApiClientConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ExternalPriceApiClientConfig { + fn from_env() -> anyhow::Result { + envy_load("external_price_api_client", "EXTERNAL_PRICE_API_CLIENT_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::external_price_api_client::{ + ExternalPriceApiClientConfig, DEFAULT_TIMEOUT_MS, + }; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_external_price_api_client_config() -> ExternalPriceApiClientConfig { + ExternalPriceApiClientConfig { + source: "no-op".to_string(), + base_url: Some("https://pro-api.coingecko.com".to_string()), + api_key: Some("qwerty12345".to_string()), + client_timeout_ms: DEFAULT_TIMEOUT_MS, + forced_numerator: Some(100), + forced_denominator: Some(1), + } + } + + #[test] + fn from_env_external_price_api_client() { + let mut lock = MUTEX.lock(); + let config = r#" + EXTERNAL_PRICE_API_CLIENT_SOURCE=no-op + EXTERNAL_PRICE_API_CLIENT_BASE_URL=https://pro-api.coingecko.com + EXTERNAL_PRICE_API_CLIENT_API_KEY=qwerty12345 + EXTERNAL_PRICE_API_CLIENT_FORCED_NUMERATOR=100 + EXTERNAL_PRICE_API_CLIENT_FORCED_DENOMINATOR=1 + "#; + lock.set_env(config); + + let actual = ExternalPriceApiClientConfig::from_env().unwrap(); + assert_eq!(actual, expected_external_price_api_client_config()); + } +} diff --git a/core/lib/env_config/src/lib.rs b/core/lib/env_config/src/lib.rs index bd7aa035b68b..789f6f8be2fd 100644 --- a/core/lib/env_config/src/lib.rs +++ b/core/lib/env_config/src/lib.rs @@ -23,6 +23,7 @@ mod utils; mod base_token_adjuster; mod da_dispatcher; +mod external_price_api_client; mod genesis; #[cfg(test)] mod test_utils; diff --git a/core/lib/external_price_api/Cargo.toml b/core/lib/external_price_api/Cargo.toml index c75ff5851d75..40ff295fbced 100644 --- a/core/lib/external_price_api/Cargo.toml +++ b/core/lib/external_price_api/Cargo.toml @@ -12,6 +12,14 @@ categories.workspace = true [dependencies] async-trait.workspace = true anyhow.workspace = true +url.workspace = true +bigdecimal.workspace = true +chrono.workspace = true +serde.workspace = true +reqwest.workspace = true +fraction.workspace = true +rand.workspace = true zksync_config.workspace = true zksync_types.workspace = true +tokio.workspace = true diff --git a/core/lib/external_price_api/src/coingecko_api.rs b/core/lib/external_price_api/src/coingecko_api.rs new file mode 100644 index 000000000000..8fa7514b3684 --- /dev/null +++ b/core/lib/external_price_api/src/coingecko_api.rs @@ -0,0 +1,112 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use chrono::Utc; +use reqwest; +use serde::{Deserialize, Serialize}; +use url::Url; +use zksync_config::configs::ExternalPriceApiClientConfig; +use zksync_types::{base_token_ratio::BaseTokenAPIRatio, Address}; + +use crate::{address_to_string, utils::get_fraction, PriceAPIClient}; + +#[derive(Debug)] +pub struct CoinGeckoPriceAPIClient { + base_url: Url, + client: reqwest::Client, +} + +const DEFAULT_COINGECKO_API_URL: &str = "https://pro-api.coingecko.com"; +const COINGECKO_AUTH_HEADER: &str = "x-cg-pro-api-key"; +const ETH_ID: &str = "eth"; + +impl CoinGeckoPriceAPIClient { + pub fn new(config: ExternalPriceApiClientConfig) -> Self { + let client = if let Some(api_key) = &config.api_key { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::HeaderName::from_static(COINGECKO_AUTH_HEADER), + reqwest::header::HeaderValue::from_str(api_key) + .expect("Failed to create header value"), + ); + + reqwest::Client::builder() + .default_headers(headers) + .timeout(config.client_timeout()) + .build() + .expect("Failed to build reqwest client") + } else { + reqwest::Client::new() + }; + + let base_url = config + .base_url + .unwrap_or(DEFAULT_COINGECKO_API_URL.to_string()); + + Self { + base_url: Url::parse(&base_url).expect("Failed to parse CoinGecko URL"), + client, + } + } + + async fn get_token_price_by_address(&self, address: Address) -> anyhow::Result { + let address_str = address_to_string(&address); + let price_url = self + .base_url + .join( + format!( + "/api/v3/simple/token_price/ethereum?contract_addresses={}&vs_currencies={}", + address_str, ETH_ID + ) + .as_str(), + ) + .expect("failed to join URL path"); + + let response = self.client.get(price_url).send().await?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Http error while fetching token price. Status: {}, token_addr: {}, msg: {}", + response.status(), + address_str, + response.text().await.unwrap_or(String::new()) + )); + } + + let cg_response = response.json::().await?; + match cg_response.get_price(&address_str, Ð_ID.to_string()) { + Some(&price) => Ok(price), + None => Err(anyhow::anyhow!( + "Price not found for token: {}", + address_str + )), + } + } +} + +#[async_trait] +impl PriceAPIClient for CoinGeckoPriceAPIClient { + async fn fetch_ratio(&self, token_address: Address) -> anyhow::Result { + let base_token_in_eth = self.get_token_price_by_address(token_address).await?; + let (numerator, denominator) = get_fraction(base_token_in_eth); + + return Ok(BaseTokenAPIRatio { + numerator, + denominator, + ratio_timestamp: Utc::now(), + }); + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CoinGeckoPriceResponse { + #[serde(flatten)] + pub(crate) prices: HashMap>, +} + +impl CoinGeckoPriceResponse { + fn get_price(&self, address: &String, currency: &String) -> Option<&f64> { + self.prices + .get(address) + .and_then(|price| price.get(currency)) + } +} diff --git a/core/lib/external_price_api/src/forced_price_client.rs b/core/lib/external_price_api/src/forced_price_client.rs new file mode 100644 index 000000000000..f4b8d72b8b2c --- /dev/null +++ b/core/lib/external_price_api/src/forced_price_client.rs @@ -0,0 +1,62 @@ +use std::num::NonZeroU64; + +use async_trait::async_trait; +use rand::Rng; +use zksync_config::configs::ExternalPriceApiClientConfig; +use zksync_types::{base_token_ratio::BaseTokenAPIRatio, Address}; + +use crate::PriceAPIClient; + +// Struct for a a forced price "client" (conversion ratio is always a configured "forced" ratio). +#[derive(Debug, Clone)] +pub struct ForcedPriceClient { + ratio: BaseTokenAPIRatio, +} + +impl ForcedPriceClient { + pub fn new(config: ExternalPriceApiClientConfig) -> Self { + let numerator = config + .forced_numerator + .expect("forced price client started with no forced numerator"); + let denominator = config + .forced_denominator + .expect("forced price client started with no forced denominator"); + + Self { + ratio: BaseTokenAPIRatio { + numerator: NonZeroU64::new(numerator).unwrap(), + denominator: NonZeroU64::new(denominator).unwrap(), + ratio_timestamp: chrono::Utc::now(), + }, + } + } +} + +#[async_trait] +impl PriceAPIClient for ForcedPriceClient { + // Returns a ratio which is 10% higher or lower than the configured forced ratio. + async fn fetch_ratio(&self, _token_address: Address) -> anyhow::Result { + let mut rng = rand::thread_rng(); + + let numerator_range = ( + (self.ratio.numerator.get() as f64 * 0.9).round() as u64, + (self.ratio.numerator.get() as f64 * 1.1).round() as u64, + ); + + let denominator_range = ( + (self.ratio.denominator.get() as f64 * 0.9).round() as u64, + (self.ratio.denominator.get() as f64 * 1.1).round() as u64, + ); + + let new_numerator = rng.gen_range(numerator_range.0..=numerator_range.1); + let new_denominator = rng.gen_range(denominator_range.0..=denominator_range.1); + + let adjusted_ratio = BaseTokenAPIRatio { + numerator: NonZeroU64::new(new_numerator).unwrap_or(self.ratio.numerator), + denominator: NonZeroU64::new(new_denominator).unwrap_or(self.ratio.denominator), + ratio_timestamp: chrono::Utc::now(), + }; + + Ok(adjusted_ratio) + } +} diff --git a/core/lib/external_price_api/src/lib.rs b/core/lib/external_price_api/src/lib.rs index 4128c0f231f8..e86279dbe850 100644 --- a/core/lib/external_price_api/src/lib.rs +++ b/core/lib/external_price_api/src/lib.rs @@ -1,3 +1,7 @@ +pub mod coingecko_api; +pub mod forced_price_client; +mod utils; + use std::fmt; use async_trait::async_trait; @@ -5,7 +9,22 @@ use zksync_types::{base_token_ratio::BaseTokenAPIRatio, Address}; /// Trait that defines the interface for a client connecting with an external API to get prices. #[async_trait] -pub trait PriceAPIClient: Sync + Send + fmt::Debug { - /// Returns the price for the input token address in $USD. - async fn fetch_price(&self, token_address: Address) -> anyhow::Result; +pub trait PriceAPIClient: Sync + Send + fmt::Debug + 'static { + /// Returns the BaseToken<->ETH ratio for the input token address. + async fn fetch_ratio(&self, token_address: Address) -> anyhow::Result; +} + +// Struct for a no-op PriceAPIClient (conversion ratio is always 1:1). +#[derive(Debug, Clone)] +pub struct NoOpPriceAPIClient; + +#[async_trait] +impl PriceAPIClient for NoOpPriceAPIClient { + async fn fetch_ratio(&self, _token_address: Address) -> anyhow::Result { + Ok(BaseTokenAPIRatio::default()) + } +} + +fn address_to_string(address: &Address) -> String { + format!("{:#x}", address) } diff --git a/core/lib/external_price_api/src/utils.rs b/core/lib/external_price_api/src/utils.rs new file mode 100644 index 000000000000..879be44e1737 --- /dev/null +++ b/core/lib/external_price_api/src/utils.rs @@ -0,0 +1,15 @@ +use std::num::NonZeroU64; + +use fraction::Fraction; + +/// Using the base token price and eth price, calculate the fraction of the base token to eth. +pub fn get_fraction(ratio_f64: f64) -> (NonZeroU64, NonZeroU64) { + let rate_fraction = Fraction::from(ratio_f64); + + let numerator = NonZeroU64::new(*rate_fraction.numer().expect("numerator is empty")) + .expect("numerator is zero"); + let denominator = NonZeroU64::new(*rate_fraction.denom().expect("denominator is empty")) + .expect("denominator is zero"); + + (numerator, denominator) +} diff --git a/core/lib/protobuf_config/src/external_price_api_client.rs b/core/lib/protobuf_config/src/external_price_api_client.rs new file mode 100644 index 000000000000..cd16957d55ad --- /dev/null +++ b/core/lib/protobuf_config/src/external_price_api_client.rs @@ -0,0 +1,32 @@ +use zksync_config::configs::{self}; +use zksync_protobuf::ProtoRepr; + +use crate::proto::external_price_api_client as proto; + +impl ProtoRepr for proto::ExternalPriceApiClient { + type Type = configs::external_price_api_client::ExternalPriceApiClientConfig; + + fn read(&self) -> anyhow::Result { + Ok( + configs::external_price_api_client::ExternalPriceApiClientConfig { + source: self.source.clone().expect("source"), + client_timeout_ms: self.client_timeout_ms.expect("client_timeout_ms"), + base_url: self.base_url.clone(), + api_key: self.api_key.clone(), + forced_numerator: self.forced_numerator, + forced_denominator: self.forced_denominator, + }, + ) + } + + fn build(this: &Self::Type) -> Self { + Self { + source: Some(this.source.clone()), + base_url: this.base_url.clone(), + api_key: this.api_key.clone(), + client_timeout_ms: Some(this.client_timeout_ms), + forced_numerator: this.forced_numerator, + forced_denominator: this.forced_denominator, + } + } +} diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs index 9361c02b18d7..44ce9d8d1eba 100644 --- a/core/lib/protobuf_config/src/general.rs +++ b/core/lib/protobuf_config/src/general.rs @@ -54,6 +54,8 @@ impl ProtoRepr for proto::GeneralConfig { pruning: read_optional_repr(&self.pruning).context("pruning")?, snapshot_recovery: read_optional_repr(&self.snapshot_recovery) .context("snapshot_recovery")?, + external_price_api_client_config: read_optional_repr(&self.external_price_api_client) + .context("external_price_api_client")?, }) } @@ -99,6 +101,10 @@ impl ProtoRepr for proto::GeneralConfig { pruning: this.pruning.as_ref().map(ProtoRepr::build), core_object_store: this.core_object_store.as_ref().map(ProtoRepr::build), base_token_adjuster: this.base_token_adjuster.as_ref().map(ProtoRepr::build), + external_price_api_client: this + .external_price_api_client_config + .as_ref() + .map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index d525c03cdb59..839f3e3cf8ca 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -29,6 +29,7 @@ mod pruning; mod secrets; mod snapshots_creator; +mod external_price_api_client; mod snapshot_recovery; #[cfg(test)] mod tests; diff --git a/core/lib/protobuf_config/src/proto/config/external_price_api_client.proto b/core/lib/protobuf_config/src/proto/config/external_price_api_client.proto new file mode 100644 index 000000000000..f47e35782e60 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/external_price_api_client.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package zksync.config.external_price_api_client; + +message ExternalPriceApiClient { + optional string source = 1; + optional string base_url = 2; + optional string api_key = 3; + optional uint64 client_timeout_ms = 4; + optional uint64 forced_numerator = 5; + optional uint64 forced_denominator = 6; +} diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto index a749fe37b238..be64f7bb97ee 100644 --- a/core/lib/protobuf_config/src/proto/config/general.proto +++ b/core/lib/protobuf_config/src/proto/config/general.proto @@ -20,6 +20,7 @@ import "zksync/config/snapshot_recovery.proto"; import "zksync/config/pruning.proto"; import "zksync/config/object_store.proto"; import "zksync/config/base_token_adjuster.proto"; +import "zksync/config/external_price_api_client.proto"; message GeneralConfig { optional config.database.Postgres postgres = 1; @@ -50,4 +51,5 @@ message GeneralConfig { optional config.da_dispatcher.DataAvailabilityDispatcher da_dispatcher = 38; optional config.base_token_adjuster.BaseTokenAdjuster base_token_adjuster = 39; optional config.vm_runner.BasicWitnessInputProducer basic_witness_input_producer = 40; + optional config.external_price_api_client.ExternalPriceApiClient external_price_api_client = 41; } diff --git a/core/lib/types/src/base_token_ratio.rs b/core/lib/types/src/base_token_ratio.rs index 0782e67ab4b0..019a84dcb706 100644 --- a/core/lib/types/src/base_token_ratio.rs +++ b/core/lib/types/src/base_token_ratio.rs @@ -13,10 +13,20 @@ pub struct BaseTokenRatio { } /// Struct to represent API response containing denominator, numerator, and timestamp. -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq)] pub struct BaseTokenAPIRatio { pub numerator: NonZeroU64, pub denominator: NonZeroU64, - /// Either the timestamp of the quote or the timestamp of the request. + // Either the timestamp of the quote or the timestamp of the request. pub ratio_timestamp: DateTime, } + +impl Default for BaseTokenAPIRatio { + fn default() -> Self { + Self { + numerator: NonZeroU64::new(1).unwrap(), + denominator: NonZeroU64::new(1).unwrap(), + ratio_timestamp: Utc::now(), + } + } +} diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index 65b7d1e43200..c05999cfa512 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -12,10 +12,10 @@ use zksync_config::{ house_keeper::HouseKeeperConfig, vm_runner::BasicWitnessInputProducerConfig, wallets::{AddressWallet, EthSender, StateKeeper, Wallet, Wallets}, - CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, - FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, - GeneralConfig, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, - ProtectiveReadsWriterConfig, PruningConfig, SnapshotRecoveryConfig, + CommitmentGeneratorConfig, ExternalPriceApiClientConfig, FriProofCompressorConfig, + FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, + FriWitnessVectorGeneratorConfig, GeneralConfig, ObservabilityConfig, PrometheusConfig, + ProofDataHandlerConfig, ProtectiveReadsWriterConfig, PruningConfig, SnapshotRecoveryConfig, }, ApiConfig, BaseTokenAdjusterConfig, ContractVerifierConfig, DADispatcherConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, ObjectStoreConfig, PostgresConfig, @@ -73,6 +73,7 @@ pub struct TempConfigStore { pub commitment_generator: Option, pub pruning: Option, pub snapshot_recovery: Option, + pub external_price_api_client_config: Option, } impl TempConfigStore { @@ -106,6 +107,7 @@ impl TempConfigStore { commitment_generator: self.commitment_generator.clone(), snapshot_recovery: self.snapshot_recovery.clone(), pruning: self.pruning.clone(), + external_price_api_client_config: self.external_price_api_client_config.clone(), } } diff --git a/core/node/base_token_adjuster/Cargo.toml b/core/node/base_token_adjuster/Cargo.toml index 7e5c5bcaae43..34a38b2bbf77 100644 --- a/core/node/base_token_adjuster/Cargo.toml +++ b/core/node/base_token_adjuster/Cargo.toml @@ -14,6 +14,7 @@ categories.workspace = true zksync_dal.workspace = true zksync_config.workspace = true zksync_types.workspace = true +zksync_external_price_api.workspace = true tokio = { workspace = true, features = ["time"] } anyhow.workspace = true diff --git a/core/node/base_token_adjuster/src/base_token_ratio_persister.rs b/core/node/base_token_adjuster/src/base_token_ratio_persister.rs index b730737b992f..8c94b19e0179 100644 --- a/core/node/base_token_adjuster/src/base_token_ratio_persister.rs +++ b/core/node/base_token_adjuster/src/base_token_ratio_persister.rs @@ -1,28 +1,39 @@ -use std::{fmt::Debug, num::NonZero}; +use std::{fmt::Debug, sync::Arc, time::Duration}; use anyhow::Context as _; -use chrono::Utc; -use tokio::sync::watch; +use tokio::{sync::watch, time::sleep}; use zksync_config::configs::base_token_adjuster::BaseTokenAdjusterConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; -use zksync_types::base_token_ratio::BaseTokenAPIRatio; +use zksync_external_price_api::PriceAPIClient; +use zksync_types::{base_token_ratio::BaseTokenAPIRatio, Address}; #[derive(Debug, Clone)] pub struct BaseTokenRatioPersister { pool: ConnectionPool, config: BaseTokenAdjusterConfig, + base_token_address: Address, + price_api_client: Arc, } impl BaseTokenRatioPersister { - pub fn new(pool: ConnectionPool, config: BaseTokenAdjusterConfig) -> Self { - Self { pool, config } + pub fn new( + pool: ConnectionPool, + config: BaseTokenAdjusterConfig, + base_token_address: Address, + price_api_client: Arc, + ) -> Self { + Self { + pool, + config, + base_token_address, + price_api_client, + } } /// Main loop for the base token ratio persister. /// Orchestrates fetching a new ratio, persisting it, and conditionally updating the L1 with it. pub async fn run(&mut self, mut stop_receiver: watch::Receiver) -> anyhow::Result<()> { let mut timer = tokio::time::interval(self.config.price_polling_interval()); - let pool = self.pool.clone(); while !*stop_receiver.borrow_and_update() { tokio::select! { @@ -30,33 +41,61 @@ impl BaseTokenRatioPersister { _ = stop_receiver.changed() => break, } - let new_ratio = self.fetch_new_ratio().await?; - self.persist_ratio(&new_ratio, &pool).await?; - // TODO(PE-128): Update L1 ratio + if let Err(err) = self.loop_iteration().await { + return Err(err) + .context("Failed to execute a base_token_ratio_persister loop iteration"); + } } tracing::info!("Stop signal received, base_token_ratio_persister is shutting down"); Ok(()) } - // TODO (PE-135): Use real API client to fetch new ratio through self.PriceAPIClient & mock for tests. - // For now, these are hard coded dummy values. - async fn fetch_new_ratio(&self) -> anyhow::Result { - let ratio_timestamp = Utc::now(); + async fn loop_iteration(&self) -> anyhow::Result<()> { + // TODO(PE-148): Consider shifting retry upon adding external API redundancy. + let new_ratio = self.retry_fetch_ratio().await?; + + self.persist_ratio(new_ratio).await?; + // TODO(PE-128): Update L1 ratio - Ok(BaseTokenAPIRatio { - numerator: NonZero::new(1).unwrap(), - denominator: NonZero::new(100000).unwrap(), - ratio_timestamp, - }) + Ok(()) + } + + async fn retry_fetch_ratio(&self) -> anyhow::Result { + let sleep_duration = Duration::from_secs(1); + let max_retries = 5; + let mut attempts = 0; + + loop { + match self + .price_api_client + .fetch_ratio(self.base_token_address) + .await + { + Ok(ratio) => { + return Ok(ratio); + } + Err(err) if attempts < max_retries => { + attempts += 1; + tracing::warn!( + "Attempt {}/{} to fetch ratio from coingecko failed with err: {}. Retrying...", + attempts, + max_retries, + err + ); + sleep(sleep_duration).await; + } + Err(err) => { + return Err(err) + .context("Failed to fetch base token ratio after multiple attempts"); + } + } + } } - async fn persist_ratio( - &self, - api_price: &BaseTokenAPIRatio, - pool: &ConnectionPool, - ) -> anyhow::Result { - let mut conn = pool + async fn persist_ratio(&self, api_ratio: BaseTokenAPIRatio) -> anyhow::Result { + let mut conn = self + .pool .connection_tagged("base_token_ratio_persister") .await .context("Failed to obtain connection to the database")?; @@ -64,9 +103,9 @@ impl BaseTokenRatioPersister { let id = conn .base_token_dal() .insert_token_ratio( - api_price.numerator, - api_price.denominator, - &api_price.ratio_timestamp.naive_utc(), + api_ratio.numerator, + api_ratio.denominator, + &api_ratio.ratio_timestamp.naive_utc(), ) .await .context("Failed to insert base token ratio into the database")?; diff --git a/core/node/node_framework/Cargo.toml b/core/node/node_framework/Cargo.toml index 0edbe680ca83..b6a4bd227b4f 100644 --- a/core/node/node_framework/Cargo.toml +++ b/core/node/node_framework/Cargo.toml @@ -51,6 +51,7 @@ zksync_vm_runner.workspace = true zksync_node_db_pruner.workspace = true zksync_base_token_adjuster.workspace = true zksync_node_storage_init.workspace = true +zksync_external_price_api.workspace = true pin-project-lite.workspace = true tracing.workspace = true diff --git a/core/node/node_framework/src/implementations/layers/base_token_ratio_persister.rs b/core/node/node_framework/src/implementations/layers/base_token/base_token_ratio_persister.rs similarity index 63% rename from core/node/node_framework/src/implementations/layers/base_token_ratio_persister.rs rename to core/node/node_framework/src/implementations/layers/base_token/base_token_ratio_persister.rs index 9bf1786f6bbc..d15f9bea0e25 100644 --- a/core/node/node_framework/src/implementations/layers/base_token_ratio_persister.rs +++ b/core/node/node_framework/src/implementations/layers/base_token/base_token_ratio_persister.rs @@ -1,8 +1,11 @@ use zksync_base_token_adjuster::BaseTokenRatioPersister; -use zksync_config::configs::base_token_adjuster::BaseTokenAdjusterConfig; +use zksync_config::{configs::base_token_adjuster::BaseTokenAdjusterConfig, ContractsConfig}; use crate::{ - implementations::resources::pools::{MasterPool, PoolResource}, + implementations::resources::{ + pools::{MasterPool, PoolResource}, + price_api_client::PriceAPIClientResource, + }, service::StopReceiver, task::{Task, TaskId}, wiring_layer::{WiringError, WiringLayer}, @@ -16,12 +19,15 @@ use crate::{ #[derive(Debug)] pub struct BaseTokenRatioPersisterLayer { config: BaseTokenAdjusterConfig, + contracts_config: ContractsConfig, } #[derive(Debug, FromContext)] #[context(crate = crate)] pub struct Input { pub master_pool: PoolResource, + #[context(default)] + pub price_api_client: PriceAPIClientResource, } #[derive(Debug, IntoContext)] @@ -32,8 +38,11 @@ pub struct Output { } impl BaseTokenRatioPersisterLayer { - pub fn new(config: BaseTokenAdjusterConfig) -> Self { - Self { config } + pub fn new(config: BaseTokenAdjusterConfig, contracts_config: ContractsConfig) -> Self { + Self { + config, + contracts_config, + } } } @@ -48,7 +57,20 @@ impl WiringLayer for BaseTokenRatioPersisterLayer { async fn wire(self, input: Self::Input) -> Result { let master_pool = input.master_pool.get().await?; - let persister = BaseTokenRatioPersister::new(master_pool, self.config); + + let price_api_client = input.price_api_client; + let base_token_addr = self + .contracts_config + .base_token_addr + .expect("base token address is not set"); + + let persister = BaseTokenRatioPersister::new( + master_pool, + self.config, + base_token_addr, + price_api_client.0, + ); + Ok(Output { persister }) } } diff --git a/core/node/node_framework/src/implementations/layers/base_token_ratio_provider.rs b/core/node/node_framework/src/implementations/layers/base_token/base_token_ratio_provider.rs similarity index 100% rename from core/node/node_framework/src/implementations/layers/base_token_ratio_provider.rs rename to core/node/node_framework/src/implementations/layers/base_token/base_token_ratio_provider.rs diff --git a/core/node/node_framework/src/implementations/layers/base_token/coingecko_client.rs b/core/node/node_framework/src/implementations/layers/base_token/coingecko_client.rs new file mode 100644 index 000000000000..14ab568c2f3a --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/base_token/coingecko_client.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use zksync_config::configs::ExternalPriceApiClientConfig; +use zksync_external_price_api::coingecko_api::CoinGeckoPriceAPIClient; + +use crate::{ + implementations::resources::price_api_client::PriceAPIClientResource, + wiring_layer::{WiringError, WiringLayer}, + IntoContext, +}; + +/// Wiring layer for `CoingeckoApiClient` +/// +/// Responsible for inserting a resource with a client to get base token prices from CoinGecko to be +/// used by the `BaseTokenRatioPersister`. +#[derive(Debug)] +pub struct CoingeckoClientLayer { + config: ExternalPriceApiClientConfig, +} + +impl CoingeckoClientLayer { + /// Identifier of used client type. + /// Can be used to choose the layer for the client based on configuration variables. + pub const CLIENT_NAME: &'static str = "coingecko"; +} + +#[derive(Debug, IntoContext)] +#[context(crate = crate)] +pub struct Output { + pub price_api_client: PriceAPIClientResource, +} + +impl CoingeckoClientLayer { + pub fn new(config: ExternalPriceApiClientConfig) -> Self { + Self { config } + } +} + +#[async_trait::async_trait] +impl WiringLayer for CoingeckoClientLayer { + type Input = (); + type Output = Output; + + fn layer_name(&self) -> &'static str { + "coingecko_api_client" + } + + async fn wire(self, _input: Self::Input) -> Result { + let cg_client = Arc::new(CoinGeckoPriceAPIClient::new(self.config)); + + Ok(Output { + price_api_client: cg_client.into(), + }) + } +} diff --git a/core/node/node_framework/src/implementations/layers/base_token/forced_price_client.rs b/core/node/node_framework/src/implementations/layers/base_token/forced_price_client.rs new file mode 100644 index 000000000000..67785dc26ed4 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/base_token/forced_price_client.rs @@ -0,0 +1,52 @@ +use std::sync::Arc; + +use zksync_config::configs::ExternalPriceApiClientConfig; +use zksync_external_price_api::forced_price_client::ForcedPriceClient; + +use crate::{ + implementations::resources::price_api_client::PriceAPIClientResource, + wiring_layer::{WiringError, WiringLayer}, + IntoContext, +}; + +/// Wiring layer for `ForcedPriceClient` +/// +/// Inserts a resource with a forced configured price to be used by the `BaseTokenRatioPersister`. +#[derive(Debug)] +pub struct ForcedPriceClientLayer { + config: ExternalPriceApiClientConfig, +} + +impl ForcedPriceClientLayer { + pub fn new(config: ExternalPriceApiClientConfig) -> Self { + Self { config } + } + + /// Identifier of used client type. + /// Can be used to choose the layer for the client based on configuration variables. + pub const CLIENT_NAME: &'static str = "forced"; +} + +#[derive(Debug, IntoContext)] +#[context(crate = crate)] +pub struct Output { + pub price_api_client: PriceAPIClientResource, +} + +#[async_trait::async_trait] +impl WiringLayer for ForcedPriceClientLayer { + type Input = (); + type Output = Output; + + fn layer_name(&self) -> &'static str { + "forced_price_client" + } + + async fn wire(self, _input: Self::Input) -> Result { + let forced_client = Arc::new(ForcedPriceClient::new(self.config)); + + Ok(Output { + price_api_client: forced_client.into(), + }) + } +} diff --git a/core/node/node_framework/src/implementations/layers/base_token/mod.rs b/core/node/node_framework/src/implementations/layers/base_token/mod.rs new file mode 100644 index 000000000000..5b58527a3d82 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/base_token/mod.rs @@ -0,0 +1,5 @@ +pub mod base_token_ratio_persister; +pub mod base_token_ratio_provider; +pub mod coingecko_client; +pub mod forced_price_client; +pub mod no_op_external_price_api_client; diff --git a/core/node/node_framework/src/implementations/layers/base_token/no_op_external_price_api_client.rs b/core/node/node_framework/src/implementations/layers/base_token/no_op_external_price_api_client.rs new file mode 100644 index 000000000000..2bf5eda798fa --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/base_token/no_op_external_price_api_client.rs @@ -0,0 +1,45 @@ +use std::sync::Arc; + +use zksync_external_price_api::NoOpPriceAPIClient; + +use crate::{ + implementations::resources::price_api_client::PriceAPIClientResource, + wiring_layer::{WiringError, WiringLayer}, + IntoContext, +}; + +/// Wiring layer for `NoOpExternalPriceApiClient` +/// +/// Inserts a resource with a no-op client to get base token prices to be used by the `BaseTokenRatioPersister`. +#[derive(Debug)] +pub struct NoOpExternalPriceApiClientLayer; + +impl NoOpExternalPriceApiClientLayer { + /// Identifier of used client type. + /// Can be used to choose the layer for the client based on configuration variables. + pub const CLIENT_NAME: &'static str = "no-op"; +} + +#[derive(Debug, IntoContext)] +#[context(crate = crate)] +pub struct Output { + pub price_api_client: PriceAPIClientResource, +} + +#[async_trait::async_trait] +impl WiringLayer for NoOpExternalPriceApiClientLayer { + type Input = (); + type Output = Output; + + fn layer_name(&self) -> &'static str { + "no_op_external_price_api_client" + } + + async fn wire(self, _input: Self::Input) -> Result { + let no_op_client = Arc::new(NoOpPriceAPIClient {}); + + Ok(Output { + price_api_client: no_op_client.into(), + }) + } +} diff --git a/core/node/node_framework/src/implementations/layers/mod.rs b/core/node/node_framework/src/implementations/layers/mod.rs index acfe6c53417a..4d2be9b11367 100644 --- a/core/node/node_framework/src/implementations/layers/mod.rs +++ b/core/node/node_framework/src/implementations/layers/mod.rs @@ -1,5 +1,4 @@ -pub mod base_token_ratio_persister; -pub mod base_token_ratio_provider; +pub mod base_token; pub mod batch_status_updater; pub mod circuit_breaker_checker; pub mod commitment_generator; diff --git a/core/node/node_framework/src/implementations/resources/mod.rs b/core/node/node_framework/src/implementations/resources/mod.rs index cbe08fadb8e9..4f82f4c3a911 100644 --- a/core/node/node_framework/src/implementations/resources/mod.rs +++ b/core/node/node_framework/src/implementations/resources/mod.rs @@ -9,6 +9,7 @@ pub mod l1_tx_params; pub mod main_node_client; pub mod object_store; pub mod pools; +pub mod price_api_client; pub mod reverter; pub mod state_keeper; pub mod sync_state; diff --git a/core/node/node_framework/src/implementations/resources/price_api_client.rs b/core/node/node_framework/src/implementations/resources/price_api_client.rs new file mode 100644 index 000000000000..6543120a26c1 --- /dev/null +++ b/core/node/node_framework/src/implementations/resources/price_api_client.rs @@ -0,0 +1,27 @@ +use std::sync::Arc; + +use zksync_external_price_api::{NoOpPriceAPIClient, PriceAPIClient}; + +use crate::resource::Resource; + +/// A resource that provides [`PriceAPIClient`] implementation to the service. +#[derive(Debug, Clone)] +pub struct PriceAPIClientResource(pub Arc); + +impl Default for PriceAPIClientResource { + fn default() -> Self { + Self(Arc::new(NoOpPriceAPIClient)) + } +} + +impl Resource for PriceAPIClientResource { + fn name() -> String { + "common/price_api_client".into() + } +} + +impl From> for PriceAPIClientResource { + fn from(provider: Arc) -> Self { + Self(provider) + } +} diff --git a/etc/env/base/external_price_api.toml b/etc/env/base/external_price_api.toml new file mode 100644 index 000000000000..635195fd7608 --- /dev/null +++ b/etc/env/base/external_price_api.toml @@ -0,0 +1,8 @@ +# Configuration for the External Price API crate + +[external_price_api_client] + +# What source to use for the external price API. Currently only options are "forced", "no-op", and "coingecko". +source = "no-op" + +client_timeout_ms = 10000 diff --git a/etc/env/base/rust.toml b/etc/env/base/rust.toml index 950e78a155a0..1bb69374ab1a 100644 --- a/etc/env/base/rust.toml +++ b/etc/env/base/rust.toml @@ -58,6 +58,7 @@ zksync_proof_fri_compressor=info,\ vise_exporter=debug,\ snapshots_creator=debug,\ zksync_base_token_adjuster=debug,\ +zksync_external_price_api=debug,\ """ # `RUST_BACKTRACE` variable diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 3a30ba9e11b7..7914ece95c70 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -299,6 +299,9 @@ prover_group: base_token_adjuster: price_polling_interval_ms: 30000 price_cache_update_interval_ms: 2000 +external_price_api_client: + source: "no-op" + client_timeout_ms: 10000 house_keeper: l1_batch_metrics_reporting_interval_ms: 10000 @@ -323,7 +326,7 @@ prometheus: observability: log_format: plain - log_directives: "zksync_node_test_utils=info,zksync_state_keeper=info,zksync_reorg_detector=info,zksync_consistency_checker=info,zksync_metadata_calculator=info,zksync_node_sync=info,zksync_node_consensus=info,zksync_contract_verification_server=info,zksync_node_api_server=info,zksync_tee_verifier_input_producer=info,zksync_node_framework=info,zksync_block_reverter=info,zksync_commitment_generator=info,zksync_node_db_pruner=info,zksync_eth_sender=info,zksync_node_fee_model=info,zksync_node_genesis=info,zksync_house_keeper=info,zksync_proof_data_handler=info,zksync_shared_metrics=info,zksync_node_test_utils=info,zksync_vm_runner=info,zksync_consensus_bft=info,zksync_consensus_network=info,zksync_consensus_storage=info,zksync_core_leftovers=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=info,zksync_db_connection=info,zksync_eth_client=info,zksync_eth_watch=debug,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=debug,zksync_utils=debug,zksync_queued_job_processor=info,zksync_types=info,zksync_mempool=debug,loadnext=info,vm=info,zksync_object_store=info,zksync_external_node=info,zksync_witness_generator=info,zksync_prover_fri=info,zksync_witness_vector_generator=info,zksync_web3_decl=debug,zksync_health_check=debug,zksync_proof_fri_compressor=info,vise_exporter=error,snapshots_creator=debug,zksync_base_token_adjuster=debug" + log_directives: "zksync_node_test_utils=info,zksync_state_keeper=info,zksync_reorg_detector=info,zksync_consistency_checker=info,zksync_metadata_calculator=info,zksync_node_sync=info,zksync_node_consensus=info,zksync_contract_verification_server=info,zksync_node_api_server=info,zksync_tee_verifier_input_producer=info,zksync_node_framework=info,zksync_block_reverter=info,zksync_commitment_generator=info,zksync_node_db_pruner=info,zksync_eth_sender=info,zksync_node_fee_model=info,zksync_node_genesis=info,zksync_house_keeper=info,zksync_proof_data_handler=info,zksync_shared_metrics=info,zksync_node_test_utils=info,zksync_vm_runner=info,zksync_consensus_bft=info,zksync_consensus_network=info,zksync_consensus_storage=info,zksync_core_leftovers=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=info,zksync_db_connection=info,zksync_eth_client=info,zksync_eth_watch=debug,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=debug,zksync_utils=debug,zksync_queued_job_processor=info,zksync_types=info,zksync_mempool=debug,loadnext=info,vm=info,zksync_object_store=info,zksync_external_node=info,zksync_witness_generator=info,zksync_prover_fri=info,zksync_witness_vector_generator=info,zksync_web3_decl=debug,zksync_health_check=debug,zksync_proof_fri_compressor=info,vise_exporter=error,snapshots_creator=debug,zksync_base_token_adjuster=debug,zksync_external_price_api=debug" sentry: url: unset panic_interval: 1800 diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 0173b4c6e04f..c186516cf3cc 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -7741,6 +7741,7 @@ dependencies = [ "rand 0.8.5", "secrecy", "serde", + "url", "zksync_basic_types", "zksync_concurrency", "zksync_consensus_utils", diff --git a/prover/config/src/lib.rs b/prover/config/src/lib.rs index 99e3ddbee8fa..9b8bf5db3af6 100644 --- a/prover/config/src/lib.rs +++ b/prover/config/src/lib.rs @@ -9,10 +9,10 @@ use zksync_config::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, BaseTokenAdjusterConfig, BasicWitnessInputProducerConfig, DADispatcherConfig, - DatabaseSecrets, FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, - FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, GeneralConfig, - ObjectStoreConfig, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, - ProtectiveReadsWriterConfig, + DatabaseSecrets, ExternalPriceApiClientConfig, FriProofCompressorConfig, FriProverConfig, + FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, + GeneralConfig, ObjectStoreConfig, ObservabilityConfig, PrometheusConfig, + ProofDataHandlerConfig, ProtectiveReadsWriterConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -57,6 +57,7 @@ fn load_env_config() -> anyhow::Result { commitment_generator: None, pruning: None, snapshot_recovery: None, + external_price_api_client_config: ExternalPriceApiClientConfig::from_env().ok(), }) } diff --git a/zk_toolbox/Cargo.lock b/zk_toolbox/Cargo.lock index d25e857582a3..29547a4b47fe 100644 --- a/zk_toolbox/Cargo.lock +++ b/zk_toolbox/Cargo.lock @@ -6405,6 +6405,7 @@ dependencies = [ "rand", "secrecy", "serde", + "url", "zksync_basic_types", "zksync_concurrency", "zksync_consensus_utils",