diff --git a/Cargo.lock b/Cargo.lock index f4c57d3d..b6659213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1199,6 +1199,7 @@ dependencies = [ "thunderdome", "tokio", "toml", + "trust-dns-resolver", "uuid 1.5.0", "web3", ] diff --git a/config-payments.toml b/config-payments.toml index 45663107..15ecd428 100644 --- a/config-payments.toml +++ b/config-payments.toml @@ -30,94 +30,6 @@ automatic-recover = false ignore-deadlines = false -[chain.ethereum] -chain-name = "Ethereum" -chain-id = 1 -currency-symbol = "ETH" -priority-fee = 1.01 -max-fee-per-gas = 40.0 -gas-left-warning-limit = 1000000 -transaction-timeout = 100 -token = { address = "0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429", symbol = "GLM" } -confirmation-blocks = 1 -block-explorer-url = "https://etherscan.io" - -[[chain.ethereum.rpc-endpoints]] -names = """ - virginia.rpc.blxrbdn.com, - public.blastapi.io, - rpc.ankr.com/eth,rpc.flashbots.net, - cloudflare-eth.com, - ethereum.publicnode.com, - rpc.payload.de - """ -endpoints = """ - https://virginia.rpc.blxrbdn.com, - https://eth-mainnet.public.blastapi.io, - https://rpc.ankr.com/eth, - https://rpc.flashbots.net, - https://cloudflare-eth.com, - https://ethereum.publicnode.com, - https://rpc.payload.de - """ -priority = 0 -max-timeout-ms = 5000 -verify-interval-secs = 60 -allowed-head-behind-secs = 120 - - -[chain.goerli] -chain-name = "Goerli" -chain-id = 5 -currency-symbol = "tETH" -priority-fee = 0.000001 -max-fee-per-gas = 10.0 -gas-left-warning-limit = 1000000 -transaction-timeout = 100 -token = { address = "0x33af15c79d64b85ba14aaffaa4577949104b22e8", symbol = "tGLM" } -multi-contract = { address = "0x7777784f803a7bf1d7f115f849d29ce5706da64a", max-at-once = 10 } -faucet-client = { max-eth-allowed = 0.009, faucet-srv = "_eth-faucet._tcp", faucet-host = "faucet.testnet.golem.network", faucet-lookup-domain = "dev.golem.network", faucet-srv-port = 4001 } -mint-contract = { address = "0xCCA41b09C1F50320bFB41BD6822BD0cdBDC7d85C", max-glm-allowed = 400 } -confirmation-blocks = 0 -block-explorer-url = "https://goerli.etherscan.io" - -[[chain.goerli.rpc-endpoints]] -names = """ - eth-goerli.g.alchemy.com/v2/demo, - eth-goerli.public.blastapi.io, - eth-goerli.api.onfinality.io/public, - rpc.goerli.mudit.blog, - endpoints.omniatech.io/v1/eth/goerli/public, - rpc.goerli.eth.gateway.fm, - goerli.blockpi.network/v1/rpc/public, - goerli.infura.io/v3/9aa, - rpc.ankr.com/eth_goerli, - ethereum-goerli-rpc.allthatnode.com, - rpc.slock.it/goerli, - www.ethercluster.com/goerli, - rpc.ankr.com/eth_goerli, -""" -endpoints = """ - https://eth-goerli.g.alchemy.com/v2/demo, - https://eth-goerli.public.blastapi.io, - https://eth-goerli.api.onfinality.io/public, - https://rpc.goerli.mudit.blog, - https://endpoints.omniatech.io/v1/eth/goerli/public, - https://rpc.goerli.eth.gateway.fm, - https://goerli.blockpi.network/v1/rpc/public, - https://goerli.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161, - https://rpc.ankr.com/eth_goerli, - https://ethereum-goerli-rpc.allthatnode.com, - https://rpc.slock.it/goerli, - https://www.ethercluster.com/goerli, - https://rpc.ankr.com/eth_goerli, -""" -priority = 0 -max-timeout-ms = 5000 -verify-interval-secs = 60 -allowed-head-behind-secs = 60 - - [chain.holesky] chain-name = "Holesky" chain-id = 17000 @@ -154,88 +66,11 @@ priority = 0 max-timeout-ms = 5000 verify-interval-secs = 60 allowed-head-behind-secs = 120 -dns-source = "https://raw.githubusercontent.com/golemfactory/erc20_payment_lib/scx1332/dns_suport/endpoints/holesky.json" +json-source = "https://raw.githubusercontent.com/golemfactory/erc20_payment_lib/scx1332/dns_suport/endpoints/holesky.json" - -[chain.mumbai] -chain-name = "Mumbai testnet" -chain-id = 80001 -currency-symbol = "tMATIC" -priority-fee = 1.0 -max-fee-per-gas = 14.0 -gas-left-warning-limit = 1000000 -transaction-timeout = 60 -token = { address = "0x2036807B0B3aaf5b1858EE822D0e111fDdac7018", symbol = "tGLM" } -multi-contract = { address = "0x800010D7d0d315DCA795110ecCf0127cBd76b89f", max-at-once = 10 } -confirmation-blocks = 1 -block-explorer-url = "https://mumbai.polygonscan.com" - -[[chain.mumbai.rpc-endpoints]] -names = """ - polygon-mumbai.g.alchemy.com/v2/demo, - polygon-testnet.public.blastapi.io, - endpoints.omniatech.io/v1/matic/mumbai/public, - rpc-mumbai.maticvigil.com, - polygontestapi.terminet.io/rpc, - matic-mumbai.chainstacklabs.com, - matic-testnet-archive-rpc.bwarelabs.com, - rpc.ankr.com/polygon_mumbai, - polygon-testnet-archive.allthatnode.com:8545, - polygon-testnet-rpc.allthatnode.com:8545, -""" -endpoints = """ - https://polygon-mumbai.g.alchemy.com/v2/demo, - https://polygon-testnet.public.blastapi.io, - https://endpoints.omniatech.io/v1/matic/mumbai/public, - https://rpc-mumbai.maticvigil.com, - https://polygontestapi.terminet.io/rpc, - https://matic-mumbai.chainstacklabs.com, - https://matic-testnet-archive-rpc.bwarelabs.com, - https://rpc.ankr.com/polygon_mumbai, - https://polygon-testnet-archive.allthatnode.com:8545, - https://polygon-testnet-rpc.allthatnode.com:8545, -""" -priority = 0 -max-timeout-ms = 5000 -allowed-head-behind-secs = 60 - - - -[chain.polygon] -chain-name = "Polygon mainnet" -chain-id = 137 -currency-symbol = "MATIC" -priority-fee = 30.111 -max-fee-per-gas = 500.0 -gas-left-warning-limit = 1000000 -transaction-timeout = 100 -token = { address = "0x0B220b82F3eA3B7F6d9A1D8ab58930C064A2b5Bf", symbol = "GLM" } -# multi-contract = { address = "0x50100d4faf5f3b09987dea36dc2eddd57a3e561b", max-at-once = 10 } -confirmation-blocks = 1 -block-explorer-url = "https://polygonscan.com" - -[[chain.polygon.rpc-endpoints]] -names = """ - polygon-rpc.com, - rpc-mainnet.maticvigil.com, - rpc-mainnet.matic.quiknode.pro, - bor.golem.network, - polygon-mainnet-archive.allthatnode.com:8545, - polygon-mainnet-rpc.allthatnode.com:8545, -""" -endpoints = """ - https://polygon-rpc.com, - https://rpc-mainnet.maticvigil.com, - https://rpc-mainnet.matic.quiknode.pro, - https://bor.golem.network, - https://polygon-mainnet-archive.allthatnode.com:8545, - https://polygon-mainnet-rpc.allthatnode.com:8545, -""" +[[chain.holesky.rpc-endpoints]] priority = 0 max-timeout-ms = 5000 +verify-interval-secs = 60 allowed-head-behind-secs = 120 - - - - - +dns-source = "holesky.rpc-node.dev.golem.network." diff --git a/crates/erc20_payment_lib/config-payments.toml b/crates/erc20_payment_lib/config-payments.toml index de3f67a7..5023e537 100644 --- a/crates/erc20_payment_lib/config-payments.toml +++ b/crates/erc20_payment_lib/config-payments.toml @@ -76,7 +76,7 @@ gas-left-warning-limit = 1000000 transaction-timeout = 100 token = { address = "0x33af15c79d64b85ba14aaffaa4577949104b22e8", symbol = "tGLM" } multi-contract = { address = "0x7777784f803a7bf1d7f115f849d29ce5706da64a", max-at-once = 10 } -faucet-client = { max-eth-allowed = 0.009, faucet-srv = "_eth-faucet._tcp", faucet-host = "faucet.testnet.golem.network", faucet-lookup-domain = "dev.golem.network", faucet-srv-port = 4001 } +faucet-client = { max-eth-allowed = 0.009, faucet-srv = "_goerli-faucet._tcp", faucet-host = "faucet.testnet.golem.network", faucet-lookup-domain = "dev.golem.network", faucet-srv-port = 4001 } mint-contract = { address = "0xCCA41b09C1F50320bFB41BD6822BD0cdBDC7d85C", max-glm-allowed = 400 } confirmation-blocks = 0 block-explorer-url = "https://goerli.etherscan.io" @@ -130,7 +130,7 @@ token = { address = "0x8888888815bf4DB87e57B609A50f938311EEd068", symbol = "tGLM multi-contract = { address = "0xAaAAAaA00E1841A63342db7188abA84BDeE236c7", max-at-once = 10 } mint-contract = { address = "0xFACe100969FF47EB58d2CF603321B581A84bcEaC", max-glm-allowed = 400 } lock-contract = { address = "0xCE78e3fe557E3754781Cc89eB7B01e940733e3F1" } -faucet-client = { max-eth-allowed = 0.009, faucet-srv = "_eth-faucet._tcp", faucet-host = "faucet.testnet.golem.network", faucet-lookup-domain = "dev.golem.network", faucet-srv-port = 4002 } +faucet-client = { max-eth-allowed = 0.009, faucet-srv = "_holesky-faucet._tcp", faucet-host = "faucet.testnet.golem.network", faucet-lookup-domain = "dev.golem.network", faucet-srv-port = 4002 } confirmation-blocks = 0 block-explorer-url = "https://holesky.etherscan.io" diff --git a/crates/erc20_payment_lib/src/config.rs b/crates/erc20_payment_lib/src/config.rs index 51752865..1b03b263 100644 --- a/crates/erc20_payment_lib/src/config.rs +++ b/crates/erc20_payment_lib/src/config.rs @@ -103,6 +103,7 @@ pub struct RpcSettings { pub names: Option, pub endpoints: Option, pub dns_source: Option, + pub json_source: Option, pub skip_validation: Option, pub backup_level: Option, pub verify_interval_secs: Option, diff --git a/crates/erc20_payment_lib/src/faucet_client.rs b/crates/erc20_payment_lib/src/faucet_client.rs index c8e31e6b..0b833ff3 100644 --- a/crates/erc20_payment_lib/src/faucet_client.rs +++ b/crates/erc20_payment_lib/src/faucet_client.rs @@ -33,12 +33,14 @@ async fn resolve_faucet_url( ) -> Result { let faucet_host = resolve_yagna_srv_record(default_lookup_domain, faucet_srv_prefix) .await - .unwrap_or_else(|_| default_faucet_host.to_string()); + .unwrap_or_else(|_| format!("{}:{}", default_faucet_host, port)); - Ok(format!("http://{faucet_host}:{port}/donate")) + println!("resolve_faucet_url: {}", faucet_host); + Ok(format!("http://{faucet_host}/donate")) } pub async fn resolve_srv_record(record: &str) -> std::io::Result { + println!("resolve_srv_record: {}", record); let resolver: TokioAsyncResolver = TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default())?; let lookup = resolver.srv_lookup(record).await?; @@ -51,11 +53,24 @@ pub async fn resolve_srv_record(record: &str) -> std::io::Result { srv.target().to_string().trim_end_matches('.'), srv.port() ); + println!("resolve_srv_record: {}", addr); log::debug!("Resolved address: {}", addr); Ok(addr) } +pub async fn resolve_txt_record(record: &str) -> std::io::Result { + let resolver: TokioAsyncResolver = + TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default())?; + let lookup = resolver.txt_lookup(record).await?; + let txt = lookup + .iter() + .next() + .ok_or_else(|| IoError::from(IoErrorKind::NotFound))?; + + Ok(txt.to_string()) +} + /// Replace domain name in URL with resolved IP address /// Hack required on windows to bypass failing resolution on Windows 10 /// Not needed when https://github.com/actix/actix-web/issues/1047 is resolved diff --git a/crates/erc20_payment_lib/src/setup.rs b/crates/erc20_payment_lib/src/setup.rs index f4e16625..eb43cb03 100644 --- a/crates/erc20_payment_lib/src/setup.rs +++ b/crates/erc20_payment_lib/src/setup.rs @@ -5,9 +5,7 @@ use crate::error::PaymentError; use crate::utils::DecimalConvExt; use crate::{err_custom_create, err_from}; use erc20_payment_lib_common::DriverEvent; -use erc20_rpc_pool::{ - Web3EndpointParams, Web3ExternalJsonSource, Web3PoolType, Web3RpcPool, Web3RpcSingleParams, -}; +use erc20_rpc_pool::{Web3EndpointParams, Web3ExternalDnsSource, Web3ExternalJsonSource, Web3PoolType, Web3RpcPool, Web3RpcSingleParams}; use rust_decimal::Decimal; use secp256k1::SecretKey; use serde::Serialize; @@ -150,6 +148,7 @@ impl PaymentSetup { for chain_config in &config.chain { let mut single_endpoints = Vec::new(); let mut json_sources = Vec::new(); + let mut dns_sources = Vec::new(); for rpc_settings in &chain_config.1.rpc_endpoints { let endpoint_names = split_string_by_coma(&rpc_settings.names).unwrap_or_default(); if let Some(endpoints) = split_string_by_coma(&rpc_settings.endpoints) { @@ -177,9 +176,27 @@ impl PaymentSetup { single_endpoints.push(endpoint); } } else if let Some(dns_source) = &rpc_settings.dns_source { + dns_sources.push(Web3ExternalDnsSource { + chain_id: chain_config.1.chain_id as u64, + dns_url: dns_source.clone(), + endpoint_params: Web3EndpointParams { + backup_level: rpc_settings.backup_level.unwrap_or(0), + skip_validation: rpc_settings.skip_validation.unwrap_or(false), + verify_interval_secs: rpc_settings.verify_interval_secs.unwrap_or(120), + max_response_time_ms: rpc_settings.max_timeout_ms.unwrap_or(10000), + max_head_behind_secs: Some( + rpc_settings.allowed_head_behind_secs.unwrap_or(120), + ), + max_number_of_consecutive_errors: rpc_settings + .max_consecutive_errors + .unwrap_or(5), + min_interval_requests_ms: rpc_settings.min_interval_ms, + }, + }); + } else if let Some(json_source) = &rpc_settings.json_source { json_sources.push(Web3ExternalJsonSource { chain_id: chain_config.1.chain_id as u64, - url: dns_source.clone(), + url: json_source.clone(), endpoint_params: Web3EndpointParams { backup_level: rpc_settings.backup_level.unwrap_or(0), skip_validation: rpc_settings.skip_validation.unwrap_or(false), @@ -200,6 +217,7 @@ impl PaymentSetup { chain_config.1.chain_id as u64, single_endpoints, json_sources, + dns_sources, mpsc_sender.as_ref().map(|s| s.downgrade()), ); diff --git a/crates/erc20_payment_lib_test/src/config_setup.rs b/crates/erc20_payment_lib_test/src/config_setup.rs index f4689852..1800823c 100644 --- a/crates/erc20_payment_lib_test/src/config_setup.rs +++ b/crates/erc20_payment_lib_test/src/config_setup.rs @@ -19,6 +19,7 @@ pub async fn create_default_config_setup(proxy_url_base: &str, proxy_key: &str) names: Some(format!("{}/web3/{}", proxy_url_base, proxy_key)), endpoints: Some(format!("{}/web3/{}", proxy_url_base, proxy_key)), dns_source: None, + json_source: None, skip_validation: None, backup_level: None, verify_interval_secs: None, diff --git a/crates/erc20_rpc_pool/Cargo.toml b/crates/erc20_rpc_pool/Cargo.toml index becb388b..1fa1bc62 100644 --- a/crates/erc20_rpc_pool/Cargo.toml +++ b/crates/erc20_rpc_pool/Cargo.toml @@ -35,6 +35,7 @@ actix-web = { workspace = true } actix-files = { workspace = true } serde_json = { workspace = true } thunderdome = { workspace = true } +trust-dns-resolver = { workspace = true } uuid = { workspace = true } env_logger = { workspace = true } reqwest = "0.11.22" diff --git a/crates/erc20_rpc_pool/src/lib.rs b/crates/erc20_rpc_pool/src/lib.rs index 9f41cc90..39422b7d 100644 --- a/crates/erc20_rpc_pool/src/lib.rs +++ b/crates/erc20_rpc_pool/src/lib.rs @@ -5,6 +5,7 @@ pub use rpc_pool::VerifyEndpointStatus; pub use rpc_pool::Web3EndpointParams; pub use rpc_pool::Web3ExternalEndpointList; pub use rpc_pool::Web3ExternalJsonSource; +pub use rpc_pool::Web3ExternalDnsSource; pub use rpc_pool::Web3PoolType; pub use rpc_pool::Web3RpcEndpoint; pub use rpc_pool::Web3RpcInfo; diff --git a/crates/erc20_rpc_pool/src/rpc_pool/external.rs b/crates/erc20_rpc_pool/src/rpc_pool/external.rs deleted file mode 100644 index 2c6ec324..00000000 --- a/crates/erc20_rpc_pool/src/rpc_pool/external.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::{Web3ExternalEndpointList, Web3RpcPool, Web3RpcSingleParams}; -use std::error::Error; -use std::sync::Arc; -use reqwest::Client; -async fn get_awc_response(url: &str) -> Result> { - let client = Client::new(); - let response = client - .get(url) - .send() - .await - .map_err(|e| format!("Error getting response from faucet {}", e))? - .text() - .await - .map_err(|e| format!("Error getting response from faucet {}", e))?; - Ok(serde_json::from_str::(&response).map_err(|e| { - format!( - "Error parsing json: {} {}", - e, - &response - ) - })?) -} - -pub async fn external_check_job(web3_pool: Arc) { - { - let mut last_external_check = web3_pool.last_external_check.lock().unwrap(); - if let Some(last_external_check) = last_external_check.as_ref() { - if last_external_check.elapsed().as_secs() < 300 { - log::error!("Last external check was less than 5 minutes ago"); - return; - } - } - last_external_check - .replace(std::time::Instant::now()) - .unwrap(); - } - let jobs = &web3_pool.external_json_sources; - - for json_source in jobs { - println!("Checking {}", json_source.url); - - let res = match get_awc_response(&json_source.url).await { - Ok(res) => res, - Err(e) => { - log::error!("Error getting response: {}", e); - continue; - } - }; - - if res.names.len() != res.urls.len() { - log::error!( - "Endpoint names and endpoints have to have same length {} != {}", - res.names.len(), - res.urls.len() - ); - } - - for (url, name) in res.urls.iter().zip(res.names) { - web3_pool.clone().add_endpoint(Web3RpcSingleParams { - chain_id: web3_pool.chain_id, - endpoint: url.clone(), - name: name.clone(), - web3_endpoint_params: json_source.endpoint_params.clone(), - }); - } - } - -} diff --git a/crates/erc20_rpc_pool/src/rpc_pool/mod.rs b/crates/erc20_rpc_pool/src/rpc_pool/mod.rs index f7fd6833..6e899bdc 100644 --- a/crates/erc20_rpc_pool/src/rpc_pool/mod.rs +++ b/crates/erc20_rpc_pool/src/rpc_pool/mod.rs @@ -9,7 +9,6 @@ mod eth_send_raw_transaction; mod eth_transaction; mod eth_transaction_count; mod eth_transaction_receipt; -mod external; mod pool; mod utils; mod verify; diff --git a/crates/erc20_rpc_pool/src/rpc_pool/pool.rs b/crates/erc20_rpc_pool/src/rpc_pool/pool.rs index cb4a784c..595e7ce7 100644 --- a/crates/erc20_rpc_pool/src/rpc_pool/pool.rs +++ b/crates/erc20_rpc_pool/src/rpc_pool/pool.rs @@ -1,16 +1,19 @@ use crate::rpc_pool::verify::{verify_endpoint, ReqStats, Web3EndpointParams, Web3RpcSingleParams}; use crate::rpc_pool::VerifyEndpointResult; use crate::Web3RpcInfo; -use chrono::{Utc}; +use chrono::Utc; use erc20_payment_lib_common::DriverEvent; use futures::future; +use reqwest::Client; use serde::{Deserialize, Serialize}; use std::collections::VecDeque; +use std::error::Error; use std::sync::{Arc, Mutex, RwLock}; use thunderdome::{Arena, Index}; +use trust_dns_resolver::config::{ResolverConfig, ResolverOpts}; +use trust_dns_resolver::TokioAsyncResolver; use web3::transports::Http; use web3::Web3; -use crate::rpc_pool::external::external_check_job; #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -27,6 +30,13 @@ pub struct Web3ExternalJsonSource { pub endpoint_params: Web3EndpointParams, } +#[derive(Debug)] +pub struct Web3ExternalDnsSource { + pub chain_id: u64, + pub dns_url: String, + pub endpoint_params: Web3EndpointParams, +} + #[derive(Debug, Serialize)] pub struct Web3RpcEndpoint { #[serde(skip)] @@ -61,6 +71,20 @@ impl Web3RpcEndpoint { } } +async fn get_awc_response(url: &str) -> Result> { + let client = Client::new(); + let response = client + .get(url) + .send() + .await + .map_err(|e| format!("Error getting response from faucet {}", e))? + .text() + .await + .map_err(|e| format!("Error getting response from faucet {}", e))?; + Ok(serde_json::from_str::(&response) + .map_err(|e| format!("Error parsing json: {} {}", e, &response))?) +} + pub type Web3PoolType = Arc>>>>; #[derive(Debug)] @@ -72,14 +96,37 @@ pub struct Web3RpcPool { pub last_success_endpoints: Arc>>, pub event_sender: Option>, pub external_json_sources: Vec, + pub external_dns_sources: Vec, pub last_external_check: Arc>>, } +#[allow(clippy::ptr_arg)] +fn split_string_by_whitespace(s: &String) -> Vec { + s.split_whitespace() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + + +pub async fn resolve_txt_record(record: &str) -> std::io::Result { + let resolver: TokioAsyncResolver = + TokioAsyncResolver::tokio(ResolverConfig::google(), ResolverOpts::default())?; + let lookup = resolver.txt_lookup(record).await?; + let txt = lookup + .iter() + .next() + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))?; + + Ok(txt.to_string()) +} + impl Web3RpcPool { pub fn new( chain_id: u64, endpoints: Vec, json_sources: Vec, + dns_sources: Vec, events: Option>, ) -> Arc { let mut web3_endpoints = Arena::new(); @@ -109,10 +156,11 @@ impl Web3RpcPool { last_success_endpoints: Arc::new(Mutex::new(VecDeque::new())), event_sender: events, external_json_sources: json_sources, + external_dns_sources: dns_sources, last_external_check: Arc::new(Mutex::new(None)), }); if !s.external_json_sources.is_empty() { - tokio::task::spawn_local(external_check_job(s.clone())); + tokio::task::spawn(s.clone().resolve_external_addresses()); } s } @@ -135,7 +183,7 @@ impl Web3RpcPool { }, }) .collect(); - Self::new(chain_id, params, Vec::new(), None) + Self::new(chain_id, params, Vec::new(), Vec::new(), None) } pub fn add_endpoint(self: Arc, endpoint: Web3RpcSingleParams) { @@ -150,7 +198,7 @@ impl Web3RpcPool { } for (_idx, el) in endpoints_locked.iter() { if el.read().unwrap().web3_rpc_params.endpoint == endpoint.endpoint { - log::error!("Endpoint {} already exists", endpoint.endpoint); + log::debug!("Endpoint {} already exists", endpoint.endpoint); return; } } @@ -219,11 +267,72 @@ impl Web3RpcPool { (extra_score_idx, extra_score) } - pub async fn resolve_external_addresses(self: Arc) {} + pub async fn resolve_external_addresses(self: Arc) { + { + let mut last_external_check = self.last_external_check.lock().unwrap(); + if let Some(last_external_check) = last_external_check.as_ref() { + if last_external_check.elapsed().as_secs() < 300 { + log::error!("Last external check was less than 5 minutes ago"); + return; + } + } + last_external_check.replace(std::time::Instant::now()); + } + + let dns_jobs = &self.external_dns_sources; + for dns_source in dns_jobs { + let record = match resolve_txt_record(&dns_source.dns_url).await { + Ok(record) => record, + Err(e) => { + log::error!("Error resolving dns entry {}: {}", &dns_source.dns_url, e); + continue; + } + }; + + let urls = split_string_by_whitespace(&record); + let names = urls.clone(); + + for (url, name) in urls.iter().zip(names) { + self.clone().add_endpoint(Web3RpcSingleParams { + chain_id: self.chain_id, + endpoint: url.clone(), + name: name.clone(), + web3_endpoint_params: dns_source.endpoint_params.clone(), + }); + } + } + let jobs = &self.external_json_sources; + + for json_source in jobs { + let res = match get_awc_response(&json_source.url).await { + Ok(res) => res, + Err(e) => { + log::error!("Error getting response: {}", e); + continue; + } + }; + + if res.names.len() != res.urls.len() { + log::error!( + "Endpoint names and endpoints have to have same length {} != {}", + res.names.len(), + res.urls.len() + ); + } + + for (url, name) in res.urls.iter().zip(res.names) { + self.clone().add_endpoint(Web3RpcSingleParams { + chain_id: self.chain_id, + endpoint: url.clone(), + name: name.clone(), + web3_endpoint_params: json_source.endpoint_params.clone(), + }); + } + } + } pub async fn choose_best_endpoints(self: Arc) -> Vec { - //to do wybrać z dnsa/urla rzeczy - external_check_job(self.clone()).await; + tokio::task::spawn(self.clone().resolve_external_addresses()); let endpoints_copy = self.endpoints.lock().unwrap().clone(); let (extra_score_idx, extra_score) = self.clone().extra_score_from_last_chosen(); diff --git a/src/main.rs b/src/main.rs index 9c008138..35d2db34 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,7 +27,7 @@ use std::str::FromStr; use crate::stats::{export_stats, run_stats}; use erc20_payment_lib::eth::check_allowance; -use erc20_payment_lib::faucet_client::faucet_donate; +use erc20_payment_lib::faucet_client::{faucet_donate, resolve_txt_record}; use erc20_payment_lib::misc::gen_private_keys; use erc20_payment_lib::runtime::{ deposit_funds, get_token_balance, mint_golem_token, remove_last_unsent_transactions, @@ -70,6 +70,17 @@ fn split_string_by_coma(s: &Option) -> Option> { }) } +#[allow(clippy::ptr_arg)] +fn split_string_by_whitespace(s: &String) -> Vec { + s.split_whitespace() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + + + + async fn main_internal() -> Result<(), PaymentError> { dotenv::dotenv().ok(); env::set_var( @@ -121,6 +132,7 @@ async fn main_internal() -> Result<(), PaymentError> { backup_level: None, max_consecutive_errors: None, dns_source: None, + json_source: None, }) .collect(); config.change_rpc_endpoints(f.1, rpcs).await?; @@ -288,11 +300,11 @@ async fn main_internal() -> Result<(), PaymentError> { }; single_endpoints.push(endpoint); } - } else if rpc_settings.dns_source.is_some() { + } else if rpc_settings.dns_source.is_some() || rpc_settings.json_source.is_some() { //process later } else { panic!( - "Endpoint has to have endpoints or dns_source {}", + "Endpoint has to have endpoints or dns_source or json_source {}", check_web3_rpc_options.chain_name, ); }; @@ -301,18 +313,51 @@ async fn main_internal() -> Result<(), PaymentError> { chain_cfg.chain_id as u64, single_endpoints, Vec::new(), + Vec::new(), None, ); for rpc_settings in &chain_cfg.rpc_endpoints { if split_string_by_coma(&rpc_settings.endpoints).is_some() { //already processed above } else if let Some(dns_source) = &rpc_settings.dns_source { + let record = resolve_txt_record(dns_source).await.map_err(|e| { + err_custom_create!("Error resolving dns entry {}: {}", dns_source, e) + })?; + + let urls = split_string_by_whitespace(&record); + let names = urls.clone(); + + for (url, name) in urls.iter().zip(names) { + log::info!("Imported from dns source: {}", name); + web3_pool.clone().add_endpoint(Web3RpcSingleParams { + chain_id: chain_cfg.chain_id as u64, + + endpoint: url.clone(), + name: name.clone(), + web3_endpoint_params: Web3EndpointParams { + backup_level: rpc_settings.backup_level.unwrap_or(0), + skip_validation: rpc_settings.skip_validation.unwrap_or(false), + verify_interval_secs: rpc_settings + .verify_interval_secs + .unwrap_or(120), + max_response_time_ms: rpc_settings.max_timeout_ms.unwrap_or(10000), + max_head_behind_secs: Some( + rpc_settings.allowed_head_behind_secs.unwrap_or(120), + ), + max_number_of_consecutive_errors: rpc_settings + .max_consecutive_errors + .unwrap_or(5), + min_interval_requests_ms: rpc_settings.min_interval_ms, + }, + }); + } + } else if let Some(json_source) = &rpc_settings.json_source { let client = awc::Client::builder() .timeout(std::time::Duration::from_secs(120)) .finish(); let response = client - .get(dns_source) + .get(json_source) .send() .await .map_err(|e| { @@ -341,7 +386,7 @@ async fn main_internal() -> Result<(), PaymentError> { } for (url, name) in res.urls.iter().zip(res.names) { - println!("{}: {}", name, url); + log::info!("Imported from json source: {}", name); web3_pool.clone().add_endpoint(Web3RpcSingleParams { chain_id: chain_cfg.chain_id as u64, diff --git a/tests/docker_03_problems/single_transfer_with_problems.rs b/tests/docker_03_problems/single_transfer_with_problems.rs index 2a7a8237..311571e0 100644 --- a/tests/docker_03_problems/single_transfer_with_problems.rs +++ b/tests/docker_03_problems/single_transfer_with_problems.rs @@ -90,6 +90,7 @@ async fn test_gas_transfer(error_probability: f64) -> Result<(), anyhow::Error> allowed_head_behind_secs: None, max_consecutive_errors: None, dns_source: None, + json_source: None, }]; set_error_probability( &proxy_url_base, proxy_key ,error_probability).await; diff --git a/tests/docker_05_rpc_pool.rs b/tests/docker_05_rpc_pool.rs index 0ef78e2b..bcee1518 100644 --- a/tests/docker_05_rpc_pool.rs +++ b/tests/docker_05_rpc_pool.rs @@ -77,6 +77,7 @@ async fn test_rpc_pool() -> Result<(), anyhow::Error> { allowed_head_behind_secs: None, max_consecutive_errors: None, dns_source: None, + json_source: None, }, ];