Fixes to replacement logic, a bit of refactor and improvements

config-payments.toml

@@ -1,6 +1,6 @@
 [engine]
-service-sleep = 1
-process-sleep = 1
+service-sleep = 10
+process-sleep = 10
 automatic-recover = false
 
 [chain.rinkeby]
@@ -61,7 +61,6 @@ transaction-timeout = 100
 token = { address = "0x0B220b82F3eA3B7F6d9A1D8ab58930C064A2b5Bf", symbol = "GLM" }
 # multi-contract = { address = "0x50100d4faf5f3b09987dea36dc2eddd57a3e561b", max-at-once = 10 }
 confirmation-blocks = 1
-allow-max-fee-greater-than-priority-fee = true
 block-explorer-url = "https://polygonscan.com"
 
 [chain.dev]

crates/erc20_payment_lib/src/config.rs

@@ -81,7 +81,7 @@ pub struct Chain {
     pub faucet_eth_amount: Option<f64>,
     pub faucet_glm_amount: Option<f64>,
     pub block_explorer_url: Option<String>,
-    pub allow_max_fee_greater_than_priority_fee: Option<bool>,
+    pub replacement_timeout: Option<f64>,
 }
 
 #[derive(Deserialize, Debug, Clone)]

crates/erc20_payment_lib/src/db/ops/chain_tx_ops.rs

@@ -33,6 +33,22 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) R
     Ok(res)
 }
 
+pub async fn get_chain_txs_by_chain_id(
+    conn: &SqlitePool,
+    chain_id: i64,
+    limit: Option<i64>,
+) -> Result<Vec<ChainTxDao>, sqlx::Error> {
+    let limit = limit.unwrap_or(i64::MAX);
+    let rows = sqlx::query_as::<_, ChainTxDao>(
+        r"SELECT * FROM chain_tx WHERE chain_id = $1 ORDER by id DESC LIMIT $2",
+    )
+    .bind(chain_id)
+    .bind(limit)
+    .fetch_all(conn)
+    .await?;
+    Ok(rows)
+}
+
 pub async fn get_chain_tx(conn: &SqlitePool, id: i64) -> Result<ChainTxDao, sqlx::Error> {
     let row = sqlx::query_as::<_, ChainTxDao>(r"SELECT * FROM chain_tx WHERE id = $1")
         .bind(id)

crates/erc20_payment_lib/src/db/ops/tx_ops.rs

@@ -8,6 +8,7 @@ pub const TRANSACTION_FILTER_PROCESSING: &str = "processing > 0 AND first_proces
 pub const TRANSACTION_FILTER_TO_PROCESS: &str = "processing > 0";
 pub const TRANSACTION_FILTER_ALL: &str = "id >= 0";
 pub const TRANSACTION_FILTER_DONE: &str = "processing = 0";
+pub const TRANSACTION_ORDER_BY_ID_AND_REPLACEMENT_ID: &str = "orig_tx_id DESC,id ASC";
 pub const TRANSACTION_ORDER_BY_CREATE_DATE: &str = "created_date ASC";
 pub const TRANSACTION_ORDER_BY_FIRST_PROCESSED_DATE_DESC: &str = "first_processed DESC";
 
@@ -124,7 +125,7 @@ pub async fn get_next_transactions_to_process(
         conn,
         Some(TRANSACTION_FILTER_TO_PROCESS),
         Some(limit),
-        Some(TRANSACTION_ORDER_BY_CREATE_DATE),
+        Some(TRANSACTION_ORDER_BY_ID_AND_REPLACEMENT_ID),
     )
     .await
 }

crates/erc20_payment_lib/src/setup.rs

@@ -38,10 +38,10 @@ pub struct ChainSetup {
     pub transaction_timeout: u64,
     pub skip_multi_contract_check: bool,
     pub confirmation_blocks: u64,
-    pub allow_max_fee_greater_than_priority_fee: Option<bool>,
     pub faucet_eth_amount: Option<U256>,
     pub faucet_glm_amount: Option<U256>,
     pub block_explorer_url: Option<String>,
+    pub replacement_timeout: Option<f64>,
 }
 
 #[derive(Serialize, Clone, Debug)]
@@ -156,11 +156,9 @@ impl PaymentSetup {
                     currency_gas_symbol: chain_config.1.currency_symbol.clone(),
                     faucet_eth_amount,
                     faucet_glm_amount,
-                    allow_max_fee_greater_than_priority_fee: chain_config
-                        .1
-                        .allow_max_fee_greater_than_priority_fee,
                     block_explorer_url: chain_config.1.block_explorer_url.clone(),
                     chain_id: chain_config.1.chain_id,
+                    replacement_timeout: chain_config.1.replacement_timeout,
                 },
             );
         }

crates/erc20_payment_lib/src/transaction.rs

@@ -537,7 +537,7 @@ pub async fn find_tx(web3: &Web3<Http>, web3_tx_dao: &mut TxDao) -> Result<bool,
 pub async fn find_receipt(
     web3: &Web3<Http>,
     web3_tx_dao: &mut TxDao,
-) -> Result<bool, PaymentError> {
+) -> Result<Option<U256>, PaymentError> {
     if let Some(tx_hash) = web3_tx_dao.tx_hash.as_ref() {
         let tx_hash = web3::types::H256::from_str(tx_hash)
             .map_err(|_err| ConversionError::from("Cannot parse tx_hash".to_string()))
@@ -558,12 +558,12 @@ pub async fn find_receipt(
                 .effective_gas_price
                 .ok_or_else(|| err_custom_create!("Effective gas price expected"))?;
             web3_tx_dao.fee_paid = Some((gas_used * effective_gas_price).to_string());
-            Ok(true)
+            Ok(Some(effective_gas_price))
         } else {
             web3_tx_dao.block_number = None;
             web3_tx_dao.chain_status = None;
             web3_tx_dao.fee_paid = None;
-            Ok(false)
+            Ok(None)
         }
     } else {
         Err(err_custom_create!("No tx hash"))

crates/erc20_payment_lib/src/utils.rs

@@ -80,7 +80,7 @@ fn compute_base(num_decimals: u32) -> rust_decimal::Decimal {
 }
 
 ///good from one gwei up to at least one billion ethers
-pub fn rust_dec_to_u256(
+fn rust_dec_to_u256(
     dec_amount: rust_decimal::Decimal,
     decimals: Option<u32>,
 ) -> Result<U256, ConversionError> {
@@ -113,7 +113,7 @@ pub fn rust_dec_to_u256(
     Ok(U256::from(u128))
 }
 
-pub fn u256_to_rust_dec(
+fn u256_to_rust_dec(
     amount: U256,
     decimals: Option<u32>,
 ) -> Result<rust_decimal::Decimal, ConversionError> {
@@ -136,6 +136,75 @@ pub fn u256_to_rust_dec(
     Ok(Decimal::from(amount.as_u128()) / dec_base)
 }
 
+fn u256_to_gwei(amount: U256) -> Result<Decimal, ConversionError> {
+    u256_to_rust_dec(amount, Some(9))
+}
+
+pub trait U256ConvExt {
+    fn to_gwei(&self) -> Result<Decimal, ConversionError>;
+    fn to_eth(&self) -> Result<Decimal, ConversionError>;
+}
+impl U256ConvExt for U256 {
+    fn to_gwei(&self) -> Result<Decimal, ConversionError> {
+        u256_to_gwei(*self)
+    }
+    fn to_eth(&self) -> Result<Decimal, ConversionError> {
+        u256_to_eth(*self)
+    }
+}
+
+pub trait StringConvExt {
+    fn to_gwei(&self) -> Result<Decimal, ConversionError>;
+    fn to_eth(&self) -> Result<Decimal, ConversionError>;
+    fn to_u256(&self) -> Result<U256, ConversionError>;
+}
+impl StringConvExt for String {
+    fn to_gwei(&self) -> Result<Decimal, ConversionError> {
+        self.to_u256()?.to_gwei()
+    }
+    fn to_eth(&self) -> Result<Decimal, ConversionError> {
+        self.to_u256()?.to_eth()
+    }
+
+    fn to_u256(&self) -> Result<U256, ConversionError> {
+        U256::from_dec_str(self).map_err(|err| {
+            ConversionError::from(format!("Invalid string when converting: {err:?}"))
+        })
+    }
+}
+
+pub trait DecimalConvExt {
+    fn to_u256_from_gwei(&self) -> Result<U256, ConversionError>;
+    fn to_u256_from_eth(&self) -> Result<U256, ConversionError>;
+}
+
+impl DecimalConvExt for Decimal {
+    fn to_u256_from_gwei(&self) -> Result<U256, ConversionError> {
+        rust_dec_to_u256(*self, Some(9))
+    }
+    fn to_u256_from_eth(&self) -> Result<U256, ConversionError> {
+        rust_dec_to_u256(*self, Some(18))
+    }
+}
+
+fn u256_to_eth(amount: U256) -> Result<Decimal, ConversionError> {
+    u256_to_rust_dec(amount, Some(18))
+}
+
+pub fn u256_eth_from_str(val: &str) -> Result<(U256, Decimal), ConversionError> {
+    let u256 = U256::from_dec_str(val)
+        .map_err(|err| ConversionError::from(format!("Invalid string when converting: {err:?}")))?;
+    let eth = u256_to_eth(u256)?;
+    Ok((u256, eth))
+}
+
+pub fn u256_gwei_from_str(val: &str) -> Result<(U256, Decimal), ConversionError> {
+    let u256 = U256::from_dec_str(val)
+        .map_err(|err| ConversionError::from(format!("Invalid string when converting: {err:?}")))?;
+    let gwei = u256_to_gwei(u256)?;
+    Ok((u256, gwei))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/erc20_payment_lib_extra/src/account_balance.rs

@@ -1,7 +1,7 @@
 use erc20_payment_lib::error::PaymentError;
 use erc20_payment_lib::eth::get_balance;
 use erc20_payment_lib::setup::PaymentSetup;
-use erc20_payment_lib::utils::u256_to_rust_dec;
+use erc20_payment_lib::utils::U256ConvExt;
 use erc20_payment_lib::{config, err_custom_create};
 use futures_util::{stream, StreamExt};
 use serde::{Deserialize, Serialize};
@@ -124,10 +124,10 @@ pub async fn account_balance(
                 log::debug!("{:#x} token: {:?}", job, token_balance);
                 let gas_balance_decimal = balance
                     .gas_balance
-                    .map(|v| u256_to_rust_dec(v, None).unwrap_or_default().to_string());
+                    .map(|v| v.to_eth().unwrap_or_default().to_string());
                 let token_balance_decimal = balance
                     .token_balance
-                    .map(|v| u256_to_rust_dec(v, None).unwrap_or_default().to_string());
+                    .map(|v| v.to_eth().unwrap_or_default().to_string());
                 let gas_balance_human = gas_balance_decimal.clone().map(|v| {
                     format!(
                         "{:.03} {}",

crates/erc20_payment_lib_test/src/config_setup.rs

@@ -29,10 +29,10 @@ pub async fn create_default_config_setup(proxy_url_base: &str, proxy_key: &str)
         }),
         transaction_timeout: 25,
         confirmation_blocks: 1,
-        allow_max_fee_greater_than_priority_fee: Some(false),
         faucet_eth_amount: Some(10.0),
         faucet_glm_amount: Some(20.0),
         block_explorer_url: Some("http://127.0.0.1:4000".to_string()),
+        replacement_timeout: Some(1.0),
     };
     let mut chain_map = BTreeMap::new();
     chain_map.insert("dev".to_string(), chain);

crates/erc20_payment_lib_test/src/durabily2.rs

@@ -9,7 +9,7 @@ use erc20_payment_lib::misc::load_private_keys;
 use erc20_payment_lib::runtime::DriverEventContent::*;
 use erc20_payment_lib::runtime::{DriverEvent, PaymentRuntime};
 use erc20_payment_lib::signer::PrivateKeySigner;
-use erc20_payment_lib::utils::u256_to_rust_dec;
+use erc20_payment_lib::utils::U256ConvExt;
 use erc20_payment_lib_extra::{generate_test_payments, GenerateOptions};
 use std::env;
 use std::path::Path;
@@ -151,12 +151,12 @@ pub async fn test_durability2(generate_count: u64, gen_interval_secs: f64, trans
     {
         // *** RESULT CHECK ***
         let (fee_paid_events, fee_paid_events_approve)  = receiver_loop.await.unwrap();
-        log::info!("fee paid from events: {}", u256_to_rust_dec(fee_paid_events, None).unwrap());
+        log::info!("fee paid from events: {}", fee_paid_events.to_eth().unwrap());
 
         let transfer_stats = get_transfer_stats(&conn, chain_id, None).await.unwrap();
         let stats_all = transfer_stats.per_sender.iter().next().unwrap().1.all.clone();
         let fee_paid_stats = stats_all.fee_paid;
-        log::info!("fee paid from stats: {}", u256_to_rust_dec(fee_paid_stats, None).unwrap());
+        log::info!("fee paid from stats: {}", fee_paid_stats.to_eth().unwrap());
 
         assert_eq!(fee_paid_events, fee_paid_stats);
 

crates/erc20_payment_lib_test/src/multi_erc20_transfer.rs

@@ -9,7 +9,7 @@ use erc20_payment_lib::misc::load_private_keys;
 use erc20_payment_lib::runtime::DriverEventContent::*;
 use erc20_payment_lib::runtime::{DriverEvent, PaymentRuntime};
 use erc20_payment_lib::signer::PrivateKeySigner;
-use erc20_payment_lib::utils::u256_to_rust_dec;
+use erc20_payment_lib::utils::U256ConvExt;
 use erc20_payment_lib_extra::{generate_test_payments, GenerateOptions};
 use std::env;
 use std::path::Path;
@@ -152,12 +152,12 @@ pub async fn test_durability(generate_count: u64, gen_interval_secs: f64, transf
     {
         // *** RESULT CHECK ***
         let (fee_paid_events, fee_paid_events_approve)  = receiver_loop.await.unwrap();
-        log::info!("fee paid from events: {}", u256_to_rust_dec(fee_paid_events, None).unwrap());
+        log::info!("fee paid from events: {}", fee_paid_events.to_eth().unwrap());
 
         let transfer_stats = get_transfer_stats(&conn, chain_id, None).await.unwrap();
         let stats_all = transfer_stats.per_sender.iter().next().unwrap().1.all.clone();
         let fee_paid_stats = stats_all.fee_paid;
-        log::info!("fee paid from stats: {}", u256_to_rust_dec(fee_paid_stats, None).unwrap());
+        log::info!("fee paid from stats: {}", fee_paid_stats.to_eth().unwrap());
 
         assert_eq!(fee_paid_events, fee_paid_stats);
 

crates/web3_test_proxy_client/src/list_txs.rs

@@ -1,5 +1,5 @@
 use crate::{get_calls, JSONRPCResult};
-use erc20_payment_lib::utils::u256_to_rust_dec;
+use erc20_payment_lib::utils::U256ConvExt;
 use web3::types::U256;
 
 /// List transactions captured by web3 proxy in human readable format
@@ -71,9 +71,7 @@ pub async fn list_transactions_human(proxy_url_base: &str, proxy_key: &str) -> V
         let result = if let Some(result_int) = result_int {
             result_int.to_string()
         } else if let Some(result_balance) = result_balance {
-            u256_to_rust_dec(result_balance, Some(18))
-                .unwrap()
-                .to_string()
+            result_balance.to_eth().unwrap().to_string()
         } else if c.method == "eth_getTransactionReceipt" {
             "details?".to_string()
         } else if call.status_code != 200 {

junk/generate_command.sh

@@ -0,0 +1 @@
+cargo run -- generate -a -c polygon -n 10 --receivers-random-pool 100000

src/main.rs

@@ -13,7 +13,6 @@ use erc20_payment_lib::db::ops::{
 };
 use erc20_payment_lib::server::*;
 use erc20_payment_lib::signer::PrivateKeySigner;
-use erc20_payment_lib::utils::rust_dec_to_u256;
 
 use erc20_payment_lib::{
     config, err_custom_create, err_from,
@@ -24,13 +23,14 @@ use erc20_payment_lib::{
 use std::env;
 use std::str::FromStr;
 
-use crate::stats::run_stats;
+use crate::stats::{export_stats, run_stats};
 use erc20_payment_lib::runtime::remove_last_unsent_transactions;
 use erc20_payment_lib::service::transaction_from_chain_and_into_db;
 use erc20_payment_lib::setup::PaymentSetup;
 use erc20_payment_lib::transaction::import_erc20_txs;
 use erc20_payment_lib_extra::{account_balance, generate_test_payments};
 
+use erc20_payment_lib::utils::DecimalConvExt;
 use std::sync::Arc;
 use structopt::StructOpt;
 use tokio::sync::Mutex;
@@ -229,7 +229,9 @@ async fn main_internal() -> Result<(), PaymentError> {
                     receiver_addr: format!("{:#x}", single_transfer_options.recipient),
                     chain_id: chain_cfg.chain_id,
                     token_addr: token,
-                    token_amount: rust_dec_to_u256(single_transfer_options.amount, Some(18))
+                    token_amount: single_transfer_options
+                        .amount
+                        .to_u256_from_eth()
                         .unwrap()
                         .to_string(),
                     create_date: Default::default(),
@@ -278,6 +280,9 @@ async fn main_internal() -> Result<(), PaymentError> {
             generate_test_payments(generate_options, &config, public_addrs, Some(conn.clone()))
                 .await?;
         }
+        PaymentCommands::ExportHistory {
+            export_history_stats_options,
+        } => export_stats(conn.clone(), export_history_stats_options, &config).await?,
         PaymentCommands::PaymentStats {
             payment_stats_options,
         } => run_stats(conn.clone(), payment_stats_options, &config).await?,
@@ -331,10 +336,21 @@ async fn main_internal() -> Result<(), PaymentError> {
                 .as_u64() as i64;
 
             //start around 30 days ago
-            let mut start_block = std::cmp::max(
-                1,
-                current_block - scan_blockchain_options.from_blocks_ago as i64,
-            );
+            let mut start_block = std::cmp::max(1, scan_blockchain_options.from_block as i64);
+
+            if scan_blockchain_options.from_block > current_block as u64 {
+                log::warn!(
+                    "From block {} is higher than current block {}, no newer data on blockchain",
+                    scan_blockchain_options.from_block,
+                    current_block
+                );
+                return Ok(());
+            }
+
+            if current_block < scan_info.last_block {
+                log::warn!("Current block {} is lower than last block from db {}, no newer data on blockchain", current_block, scan_info.last_block);
+                return Ok(());
+            }
 
             if scan_info.last_block > start_block {
                 log::info!("Start block from db is higher than start block from cli {}, using start block from db {}", start_block, scan_info.last_block);