f

crates/erc20_payment_lib/src/server.rs

@@ -16,6 +16,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
 use web3::types::Address;
+use erc20_rpc_pool::VerifyEndpointResult;
 
 pub struct ServerData {
     pub shared_state: Arc<Mutex<SharedState>>,
@@ -154,6 +155,17 @@ pub async fn rpc_pool_metrics(data: Data<Box<ServerData>>, _req: HttpRequest) ->
         metric_type: "# TYPE rpc_endpoint_success_count counter".to_string(),
         metrics: Vec::new(),
     });
+    metrics.push(MetricGroup {
+        metric_help: "# HELP rpc_endpoint_ms Endpoint validation time".to_string(),
+        metric_type: "# TYPE rpc_endpoint_ms gauge".to_string(),
+        metrics: Vec::new(),
+    });
+    metrics.push(MetricGroup {
+        metric_help: "# HELP rpc_endpoint_block_delay Time since last block head".to_string(),
+        metric_type: "# TYPE rpc_endpoint_block_delay gauge".to_string(),
+        metrics: Vec::new(),
+    });
+
 
     for (_idx, vec) in pool_ref {
         for (_idx, endpoint) in vec.iter().enumerate() {
@@ -170,14 +182,14 @@ pub async fn rpc_pool_metrics(data: Data<Box<ServerData>>, _req: HttpRequest) ->
                 params: params.clone(),
                 value: (endpoint.get_score()).to_string(),
             };
-            metrics.get_mut(0).unwrap().metrics.push(new_metric);
+            metrics[0].metrics.push(new_metric);
 
             let new_metric = Metric {
                 name: "rpc_endpoint_score_validation".into(),
                 params: params.clone(),
                 value: (endpoint.get_validation_score()).to_string(),
             };
-            metrics.get_mut(1).unwrap().metrics.push(new_metric);
+            metrics[1].metrics.push(new_metric);
 
             let new_metric = Metric {
                 name: "rpc_endpoint_error_count".into(),
@@ -188,7 +200,7 @@ pub async fn rpc_pool_metrics(data: Data<Box<ServerData>>, _req: HttpRequest) ->
                     .request_count_total_error
                     .to_string(),
             };
-            metrics.get_mut(2).unwrap().metrics.push(new_metric);
+            metrics[2].metrics.push(new_metric);
 
             let new_metric = Metric {
                 name: "rpc_endpoint_success_count".into(),
@@ -199,7 +211,31 @@ pub async fn rpc_pool_metrics(data: Data<Box<ServerData>>, _req: HttpRequest) ->
                     .request_count_total_succeeded
                     .to_string(),
             };
-            metrics.get_mut(3).unwrap().metrics.push(new_metric);
+            metrics[3].metrics.push(new_metric);
+
+
+            let (head_behind, check_time_ms) = match &endpoint
+                .web3_rpc_info
+                .verify_result {
+                    Some(VerifyEndpointResult::Ok(res)) => {
+                        (res.head_seconds_behind as i64, res.check_time_ms as i64)
+                    },
+                    _ => {(-1, -1)},
+            };
+
+            let new_metric = Metric {
+                name: "rpc_endpoint_ms".into(),
+                params: params.clone(),
+                value: check_time_ms.to_string()
+            };
+            metrics[4].metrics.push(new_metric);
+
+            let new_metric = Metric {
+                name: "rpc_endpoint_block_delay".into(),
+                params: params.clone(),
+                value: head_behind.to_string()
+            };
+            metrics[5].metrics.push(new_metric);
         }
     }
 

crates/erc20_rpc_pool/src/lib.rs

@@ -4,3 +4,5 @@ pub use rpc_pool::Web3RpcEndpoint;
 pub use rpc_pool::Web3RpcInfo;
 pub use rpc_pool::Web3RpcParams;
 pub use rpc_pool::Web3RpcPool;
+pub use rpc_pool::VerifyEndpointResult;
+pub use rpc_pool::VerifyEndpointStatus;

crates/erc20_rpc_pool/src/rpc_pool/mod.rs

@@ -15,4 +15,4 @@ mod verify;
 mod web3_error_list;
 
 pub use pool::{Web3RpcEndpoint, Web3RpcPool};
-pub use verify::{VerifyEndpointResult, Web3RpcInfo, Web3RpcParams};
+pub use verify::{VerifyEndpointResult, VerifyEndpointStatus, Web3RpcInfo, Web3RpcParams};

crates/erc20_rpc_pool/src/rpc_pool/pool.rs

@@ -27,7 +27,7 @@ impl Web3RpcEndpoint {
             + self.web3_rpc_info.penalty_from_head_behind as f64
             + self.web3_rpc_info.penalty_from_errors as f64;
 
-        let negative_score_exp = (-negative_score / 1000.0).exp();
+        let negative_score_exp = (-negative_score / 500.0).exp();
         //negative_score_exp should be in 0 to 1 range
         negative_score_exp * 75.0 + self.web3_rpc_info.bonus_from_last_chosen as f64
     }
@@ -37,7 +37,7 @@ impl Web3RpcEndpoint {
         }
         let negative_score = self.web3_rpc_info.penalty_from_ms as f64
             + self.web3_rpc_info.penalty_from_head_behind as f64;
-        let negative_score_exp = (-negative_score / 1000.0).exp();
+        let negative_score_exp = (-negative_score / 500.0).exp();
 
         negative_score_exp * 100.0
     }

crates/erc20_rpc_pool/src/rpc_pool/verify/verify_info.rs

@@ -10,8 +10,8 @@ pub struct VerifyEndpointParams {
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 pub struct VerifyEndpointStatus {
-    pub(crate) head_seconds_behind: u64,
-    pub(crate) check_time_ms: u64,
+    pub head_seconds_behind: u64,
+    pub check_time_ms: u64,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]

scenarios/rpc_pool_balancer/config-payments-goerli.toml

@@ -0,0 +1,149 @@
+[engine]
+# proces interval (in seconds) is to set how often we want to recheck transaction status
+# minimum 1 second, sensible maximum around 60 seconds
+process-interval = 5
+# proces interval after send (in seconds) is to set how long to wait after sending transaction before checking for confirmation
+# sensible minimum 20 seconds, sensible maximum around 60 seconds
+process-interval-after-send = 5
+# proces interval after error (in seconds) is to set how long to wait after encountering error before trying again
+# minimum 1 second, sensible maximum around 60 seconds
+process-interval-after-error = 5
+
+# proces interval after missing gas or token (in seconds)
+# it is starting with checking every process-interval-after-no-gas-or-token-start
+# and then increasing by multiplying by process-interval-after-no-gas-or-token-increase
+# up to process-interval-after-no-gas-or-token-max
+process-interval-after-no-gas-or-token-start = 5
+process-interval-after-no-gas-or-token-max = 5
+process-interval-after-no-gas-or-token-increase = 1.5
+
+# report alive interval (in seconds) is to set how often we want to report that we are alive
+# minimum 1 second, maximum is capped by gather-interval
+report-alive-interval = 10
+# gather interval (in seconds) is to set how often payments are gathered
+# minimum 1 second, no maximum limit
+gather-interval = 60
+# gather payments on payment driver start (otherwise wait for first gather-interval)
+gather-at-start = true
+automatic-recover = false
+# set to true to not respect deadlines attached to payments
+ignore-deadlines = false
+
+[chain.goerli]
+chain-name = "Goerli"
+chain-id = 5
+currency-symbol = "tETH"
+priority-fee = 0.000001
+max-fee-per-gas = 10.0
+gas-left-warning-limit = 1000000
+transaction-timeout = 100
+token = { address = "0x33af15c79d64b85ba14aaffaa4577949104b22e8", symbol = "tGLM" }
+multi-contract = { address = "0x7777784f803a7bf1d7f115f849d29ce5706da64a", max-at-once = 10 }
+faucet-client = { max-eth-allowed = 0.009, faucet-srv = "_eth-faucet._tcp", faucet-host = "faucet.testnet.golem.network", faucet-lookup-domain = "dev.golem.network", faucet-srv-port = 4001 }
+mint-contract = { address = "0xCCA41b09C1F50320bFB41BD6822BD0cdBDC7d85C", max-glm-allowed = 400 }
+confirmation-blocks = 0
+block-explorer-url = "https://goerli.etherscan.io"
+
+[[chain.goerli.rpc-endpoints]]
+name = "eth-goerli.g.alchemy.com"
+endpoint = "https://eth-goerli.g.alchemy.com/v2/demo"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "eth-goerli.public.blastapi.io"
+endpoint = "https://eth-goerli.public.blastapi.io"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "eth-goerli.api.onfinality.io/public"
+endpoint = "https://eth-goerli.api.onfinality.io/public"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "rpc.goerli.mudit.blog"
+endpoint = "https://rpc.goerli.mudit.blog"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "endpoints.omniatech.io/v1/eth/goerli/public"
+endpoint = "https://endpoints.omniatech.io/v1/eth/goerli/public"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "rpc.goerli.eth.gateway.fm"
+endpoint = "https://rpc.goerli.eth.gateway.fm"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "goerli.blockpi.network/v1/rpc/public"
+endpoint = "https://goerli.blockpi.network/v1/rpc/public"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "goerli.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161"
+endpoint = "https://goerli.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "rpc.ankr.com/eth_goerli"
+endpoint = "https://rpc.ankr.com/eth_goerli"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "ethereum-goerli-rpc.allthatnode.com"
+endpoint = "https://ethereum-goerli-rpc.allthatnode.com"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "rpc.slock.it/goerli"
+endpoint = "https://rpc.slock.it/goerli"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "www.ethercluster.com/goerli"
+endpoint = "https://www.ethercluster.com/goerli"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120
+
+[[chain.goerli.rpc-endpoints]]
+name = "rpc.ankr.com/eth_goerli"
+endpoint = "https://rpc.ankr.com/eth_goerli"
+priority = 0
+max-timeout-ms = 5000
+verify-interval-secs = 60
+allowed-head-behind-secs = 120