diff --git a/.env b/.env index 87495da4e..04a9a5f7f 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ export BITCOIN_RPC_URL=http://localhost:18443 export BITCOIN_RPC_USER=rpcuser export BITCOIN_RPC_PASS=rpcpassword +export ELECTRS_URL=http://localhost:3002 \ No newline at end of file diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 2e3056627..8a01ef8be 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -17,11 +17,6 @@ jobs: test: name: Test Suite runs-on: [self-hosted, linux] - services: - bitcoind: - image: docker.io/interlayhq/bitcoin-core:22.0 - ports: - - 18443:18443 strategy: matrix: @@ -60,9 +55,11 @@ jobs: env: RUST_LOG: info,regalloc=warn BITCOIN_RPC_URL: http://127.0.0.1:18443 - BITCOIN_RPC_USER: user - BITCOIN_RPC_PASS: pass + BITCOIN_RPC_USER: rpcuser + BITCOIN_RPC_PASS: rpcpassword + ELECTRS_URL: http://localhost:3002 run: | + docker-compose up --detach cargo test --release --workspace --features ${{ matrix.metadata }} --features uses-bitcoind - name: build run: | diff --git a/Cargo.lock b/Cargo.lock index 67dced928..05db2d8d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,7 +599,6 @@ dependencies = [ "bitcoincore-rpc", "cfg-if 1.0.0", "clap", - "esplora-btc-api", "futures 0.3.26", "hex", "hyper 0.10.16", @@ -613,6 +612,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serial_test", "sha2 0.8.2", "sp-core", "thiserror", @@ -2849,19 +2849,6 @@ dependencies = [ "sp-api", ] -[[package]] -name = "esplora-btc-api" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0309de257f421df7081bfb9d5c3167adbd2c23c3589e0b6c5a2cdbdb826be65" -dependencies = [ - "reqwest", - "serde", - "serde_derive", - "serde_json", - "url 2.3.1", -] - [[package]] name = "event-listener" version = "2.5.3" @@ -10218,7 +10205,6 @@ dependencies = [ "js-sys", "log 0.4.17", "mime 0.3.16", - "mime_guess", "native-tls", "once_cell", "percent-encoding 2.2.0", diff --git a/bitcoin/Cargo.toml b/bitcoin/Cargo.toml index 4a9cf95d9..3e7a90b0f 100644 --- a/bitcoin/Cargo.toml +++ b/bitcoin/Cargo.toml @@ -25,7 +25,6 @@ num-derive = "0.3" futures = "0.3.5" log = "0.4.0" hyper = "0.10" -esplora-btc-api = "1.0.3" sha2 = "0.8.2" cfg-if = "1.0" @@ -42,3 +41,4 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot- mockall = "0.8.1" regex = "1.4.3" rand = "0.7" +serial_test = "*" \ No newline at end of file diff --git a/bitcoin/src/electrs/error.rs b/bitcoin/src/electrs/error.rs index 0757d18e2..e208f55b9 100644 --- a/bitcoin/src/electrs/error.rs +++ b/bitcoin/src/electrs/error.rs @@ -2,7 +2,7 @@ use bitcoincore_rpc::bitcoin::{ consensus::encode::Error as BitcoinEncodeError, hashes::hex::Error as HexError, util::address::Error as BitcoinAddressError, }; -use reqwest::Error as ReqwestError; +use reqwest::{Error as ReqwestError, StatusCode}; use serde_json::Error as SerdeJsonError; use std::num::{ParseIntError, TryFromIntError}; use thiserror::Error; @@ -36,3 +36,9 @@ pub enum Error { #[error("ParseIntError: {0}")] ParseIntError(#[from] ParseIntError), } + +impl Error { + pub fn is_not_found(&self) -> bool { + matches!(self, Error::ReqwestError(err) if err.status().contains(&StatusCode::NOT_FOUND)) + } +} diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index c9e46f82f..8acfd707e 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -1,17 +1,19 @@ mod error; +mod types; pub use error::Error; +pub use types::*; use crate::{ deserialize, opcodes, serialize, Address, Block, BlockHash, BlockHeader, Builder as ScriptBuilder, FromHex, Network, OutPoint, Script, SignedAmount, ToHex, Transaction, Txid, H256, }; -use esplora_btc_api::models::{Transaction as ElectrsTransaction, Utxo as ElectrsUtxo}; use futures::future::{join_all, try_join}; use reqwest::{Client, Url}; use sha2::{Digest, Sha256}; -use std::str::FromStr; +use std::{convert::TryFrom, str::FromStr}; +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L42 const ELECTRS_TRANSACTIONS_PER_PAGE: usize = 25; // https://github.com/Blockstream/esplora/blob/master/API.md @@ -24,13 +26,6 @@ pub struct Utxo { pub value: u64, } -#[allow(dead_code)] -pub struct TxData { - pub txid: Txid, - pub raw_merkle_proof: Vec, - pub raw_tx: Vec, -} - #[derive(Debug)] pub struct TxInfo { pub confirmations: u32, @@ -75,33 +70,31 @@ impl ElectrsClient { Ok(serde_json::from_str(&body)?) } - pub(crate) async fn get_tx_hex(&self, txid: &str) -> Result { + pub(crate) async fn get_tx_hex(&self, txid: &Txid) -> Result { self.get(&format!("/tx/{txid}/hex")).await } - pub(crate) async fn get_tx_merkle_block_proof(&self, txid: &str) -> Result { + pub(crate) async fn get_tx_merkle_block_proof(&self, txid: &Txid) -> Result { self.get(&format!("/tx/{txid}/merkleblock-proof")).await } pub(crate) async fn get_raw_tx(&self, txid: &Txid) -> Result, Error> { - Ok(Vec::::from_hex(&self.get_tx_hex(&txid.to_string()).await?)?) + Ok(Vec::::from_hex(&self.get_tx_hex(txid).await?)?) } pub(crate) async fn get_raw_tx_merkle_proof(&self, txid: &Txid) -> Result, Error> { - Ok(Vec::::from_hex( - &self.get_tx_merkle_block_proof(&txid.to_string()).await?, - )?) + Ok(Vec::::from_hex(&self.get_tx_merkle_block_proof(txid).await?)?) } - pub(crate) async fn get_address_tx_history_full(&self, address: &str) -> Result, Error> { + pub async fn get_address_tx_history_full(&self, address: &str) -> Result, Error> { let mut last_seen_txid = Default::default(); - let mut ret = Vec::::new(); + let mut ret = Vec::::new(); loop { - let mut transactions: Vec = self + let mut transactions: Vec = self .get_and_decode(&format!("/address/{address}/txs/chain/{last_seen_txid}")) .await?; let page_size = transactions.len(); - last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.clone()); + last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.to_string()); ret.append(&mut transactions); if page_size < ELECTRS_TRANSACTIONS_PER_PAGE { // no further pages @@ -111,7 +104,7 @@ impl ElectrsClient { Ok(ret) } - pub(crate) async fn get_blocks_tip_height(&self) -> Result { + pub async fn get_blocks_tip_height(&self) -> Result { Ok(self.get("/blocks/tip/height").await?.parse()?) } @@ -120,11 +113,12 @@ impl ElectrsClient { Ok(BlockHash::from_str(&response)?) } - pub(crate) async fn get_block_header(&self, hash: &BlockHash) -> Result { + pub async fn get_block_header(&self, hash: &BlockHash) -> Result { let raw_block_header = Vec::::from_hex(&self.get(&format!("/block/{hash}/header")).await?)?; Ok(deserialize(&raw_block_header)?) } + // TODO: this is expensive and not strictly required by the light-client, deprecate? pub(crate) async fn get_transactions_in_block(&self, hash: &BlockHash) -> Result, Error> { let raw_txids: Vec = self.get_and_decode(&format!("/block/{hash}/txids")).await?; let txids: Vec = raw_txids @@ -146,7 +140,7 @@ impl ElectrsClient { Ok(Block { header, txdata }) } - pub(crate) async fn get_block_hash(&self, height: u32) -> Result { + pub async fn get_block_hash(&self, height: u32) -> Result { let response = self.get(&format!("/block-height/{height}")).await?; Ok(BlockHash::from_str(&response)?) } @@ -160,7 +154,7 @@ impl ElectrsClient { } pub(crate) async fn get_tx_info(&self, txid: &Txid) -> Result { - let tx: ElectrsTransaction = self.get_and_decode(&format!("/tx/{txid}")).await?; + let tx: TransactionValue = self.get_and_decode(&format!("/tx/{txid}")).await?; let tip = self.get_blocks_tip_height().await?; let (height, hash) = match tx.status.map(|status| (status.block_height, status.block_hash)) { Some((Some(height), Some(hash))) => (height as u32, hash), @@ -169,43 +163,53 @@ impl ElectrsClient { Ok(TxInfo { confirmations: tip.saturating_sub(height), height, - hash: BlockHash::from_str(&hash)?, - fee: SignedAmount::from_sat(tx.fee.unwrap_or_default() as i64), + hash, + fee: SignedAmount::from_sat(tx.fee as i64), }) } - pub(crate) async fn get_utxos_for_address(&self, address: Address) -> Result, Error> { - let utxos: Vec = self.get_and_decode(&format!("/address/{address}/utxo")).await?; - + pub async fn get_utxos_for_address(&self, address: &Address) -> Result, Error> { + let utxos: Vec = self.get_and_decode(&format!("/address/{address}/utxo")).await?; + // NOTE: includes unconfirmed mempool txs utxos .into_iter() .map(|utxo| { Ok(Utxo { outpoint: OutPoint { - txid: Txid::from_hex(&utxo.txid)?, - vout: utxo.vout as u32, + txid: utxo.txid, + vout: utxo.vout, }, - value: utxo.value as u64, + value: utxo.value, }) }) .collect::, Error>>() } pub(crate) async fn get_script_pubkey(&self, outpoint: OutPoint) -> Result { - let tx: ElectrsTransaction = self + let tx: TransactionValue = self + .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) + .await?; + Ok(tx + .vout + .get(usize::try_from(outpoint.vout)?) + .ok_or(Error::NoPrevOut)? + .scriptpubkey + .clone()) + } + + pub(crate) async fn get_prev_value(&self, outpoint: &OutPoint) -> Result { + let tx: TransactionValue = self .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) .await?; - Ok(Script::from_str( - &tx.vout - .ok_or(Error::NoPrevOut)? - .get(outpoint.vout as usize) - .ok_or(Error::NoPrevOut)? - .clone() - .scriptpubkey - .ok_or(Error::NoPrevOut)?, - )?) + Ok(tx + .vout + .get(usize::try_from(outpoint.vout)?) + .ok_or(Error::NoPrevOut)? + .value) } + // TODO: modify upstream to return a human-readable error + // or maybe add an endpoint for `testmempoolaccept` pub(crate) async fn send_transaction(&self, tx: Transaction) -> Result { let url = self.url.join("/tx")?; let txid = self @@ -220,8 +224,37 @@ impl ElectrsClient { Ok(Txid::from_str(&txid)?) } - #[allow(dead_code)] - pub(crate) async fn get_tx_by_op_return(&self, data: H256) -> Result, Error> { + pub(crate) async fn get_txs_by_scripthash(&self, script_hash: Vec) -> Result, Error> { + let mut last_seen_txid = Default::default(); + let mut txs = Vec::::new(); + loop { + // NOTE: includes unconfirmed txs + let mut transactions: Vec = self + .get_and_decode(&format!( + "/scripthash/{scripthash}/txs/chain/{last_seen_txid}", + scripthash = script_hash.to_hex() + )) + .await?; + let page_size = transactions.len(); + last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.to_string()); + txs.append(&mut transactions); + if page_size < ELECTRS_TRANSACTIONS_PER_PAGE { + // no further pages + break; + } + } + Ok(txs) + } + + /// Returns the *largest* payment to the `address` which is + /// greater than or equal to the specified `amount` and contains + /// an `OP_RETURN` output with `data`. + pub(crate) async fn get_tx_for_op_return( + &self, + address: Address, + amount: u128, + data: H256, + ) -> Result, Error> { let script = ScriptBuilder::new() .push_opcode(opcodes::OP_RETURN) .push_slice(data.as_bytes()) @@ -233,34 +266,29 @@ impl ElectrsClient { hasher.result().as_slice().to_vec() }; - // TODO: page this using last_seen_txid - let txs: Vec = self - .get_and_decode(&format!( - "/scripthash/{scripthash}/txs", - scripthash = script_hash.to_hex() - )) - .await?; + let txs = self.get_txs_by_scripthash(script_hash).await?; log::info!("Found {} transactions", txs.len()); - // for now, use the first tx - should probably return - // an error if there are more that one - if let Some(tx) = txs.first().cloned() { - let txid = Txid::from_str(&tx.txid)?; - log::info!("Fetching merkle proof"); - // TODO: return error if not confirmed - let raw_merkle_proof = self.get_raw_tx_merkle_proof(&txid).await?; - - log::info!("Fetching transaction"); - let raw_tx = self.get_raw_tx(&txid).await?; - - Ok(Some(TxData { - txid, - raw_merkle_proof, - raw_tx, - })) - } else { - Ok(None) + let address = address.to_string(); + for tx in txs { + let largest = tx + .vout + .iter() + .filter_map(|vout| { + if vout.scriptpubkey_address.contains(&address) { + Some(vout.value) + } else { + None + } + }) + .max() + .unwrap_or_default(); + + if largest as u128 >= amount { + return Ok(Some(tx.txid)); + } } + Ok(None) } } @@ -269,22 +297,19 @@ mod tests { use super::*; use bitcoincore_rpc::bitcoin::hashes::{hex::FromHex, sha256::Hash as Sha256Hash, Hash}; - use esplora_btc_api::apis::configuration::Configuration as ElectrsConfiguration; // TODO: mock the electrs endpoint async fn test_electrs(url: &str, script_hex: &str, expected_txid: &str) { - let config = ElectrsConfiguration { - base_path: url.to_owned(), - ..Default::default() - }; - let script_bytes = Vec::from_hex(script_hex).unwrap(); let script_hash = Sha256Hash::hash(&script_bytes); + let expected_txid = Txid::from_hex(expected_txid).unwrap(); - let txs = esplora_btc_api::apis::scripthash_api::get_txs_by_scripthash(&config, &hex::encode(script_hash)) + let electrs_client = ElectrsClient::new(Some(url.to_owned()), Network::Bitcoin).unwrap(); + let txs = electrs_client + .get_txs_by_scripthash(script_hash.to_vec()) .await .unwrap(); - assert!(txs.iter().any(|tx| { &tx.txid == expected_txid })); + assert!(txs.iter().any(|tx| tx.txid.eq(&expected_txid))); } #[tokio::test(flavor = "multi_thread")] diff --git a/bitcoin/src/electrs/types.rs b/bitcoin/src/electrs/types.rs new file mode 100644 index 000000000..aa81be13d --- /dev/null +++ b/bitcoin/src/electrs/types.rs @@ -0,0 +1,67 @@ +use crate::{BlockHash, Script, Txid}; +use serde::Deserialize; + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/util/transaction.rs#L17-L26 +#[derive(Deserialize)] +pub struct TransactionStatus { + pub confirmed: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub block_height: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub block_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub block_time: Option, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L167-L189 +#[derive(Deserialize)] +pub struct TxInValue { + pub txid: Txid, + pub vout: u32, + pub prevout: Option, + pub scriptsig: Script, + pub scriptsig_asm: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub witness: Option>, + pub is_coinbase: bool, + pub sequence: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub inner_redeemscript_asm: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub inner_witnessscript_asm: Option, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L239-L270 +#[derive(Deserialize)] +pub struct TxOutValue { + pub scriptpubkey: Script, + pub scriptpubkey_asm: String, + pub scriptpubkey_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub scriptpubkey_address: Option, + pub value: u64, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L115-L127 +#[derive(Deserialize)] +pub struct TransactionValue { + pub txid: Txid, + pub version: u32, + pub locktime: u32, + pub vin: Vec, + pub vout: Vec, + pub size: u32, + pub weight: u32, + pub fee: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L356-L396 +#[derive(Deserialize)] +pub struct UtxoValue { + pub txid: Txid, + pub vout: u32, + pub status: TransactionStatus, + pub value: u64, +} diff --git a/bitcoin/src/iter.rs b/bitcoin/src/iter.rs index a83be3fd2..311fa5c92 100644 --- a/bitcoin/src/iter.rs +++ b/bitcoin/src/iter.rs @@ -35,7 +35,7 @@ pub async fn reverse_stream_transactions( /// * `stop_height` - height of the last block the iterator will return transactions from /// * `stop_at_pruned` - whether to gracefully stop if a pruned blockchain is encountered; /// otherwise, will throw an error -pub async fn reverse_stream_in_chain_transactions( +async fn reverse_stream_in_chain_transactions( rpc: &DynBitcoinCoreApi, stop_height: u32, ) -> impl Stream> + Send + Unpin + '_ { @@ -62,7 +62,7 @@ pub async fn reverse_stream_in_chain_transactions( /// * `stop_height` - height of the last block the stream will return /// * `stop_at_pruned` - whether to gracefully stop if a pruned blockchain is encountered; /// otherwise, will throw an error -pub async fn reverse_stream_blocks( +async fn reverse_stream_blocks( rpc: &DynBitcoinCoreApi, stop_height: u32, ) -> impl Stream> + Unpin + '_ { @@ -202,6 +202,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; @@ -262,6 +263,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, Error>; } } diff --git a/bitcoin/src/lib.rs b/bitcoin/src/lib.rs index 319d99b34..cba435726 100644 --- a/bitcoin/src/lib.rs +++ b/bitcoin/src/lib.rs @@ -1,4 +1,5 @@ #![feature(int_roundings)] +#![feature(option_result_contains)] pub mod cli; pub mod light; @@ -124,6 +125,10 @@ pub struct TransactionMetadata { #[async_trait] pub trait BitcoinCoreApi { + fn is_full_node(&self) -> bool { + true + } + fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; @@ -200,10 +205,12 @@ pub trait BitcoinCoreApi { async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, Error>; } -struct LockedTransaction { - transaction: Transaction, +pub struct LockedTransaction { + pub transaction: Transaction, recipient: String, _lock: Option>, } @@ -1012,8 +1019,6 @@ impl BitcoinCoreApi for BitcoinCore { } }; tx.vout - .as_ref() - .unwrap_or(&vec![]) .iter() .any(|output| matches!(&output.scriptpubkey_address, Some(addr) if addr == &address)) }); @@ -1070,6 +1075,11 @@ impl BitcoinCoreApi for BitcoinCore { let fee_rate = fee.checked_div(vsize).ok_or(Error::ArithmeticError)?; Ok(SatPerVbyte(fee_rate.try_into()?)) } + + async fn get_tx_for_op_return(&self, _address: Address, _amount: u128, _data: H256) -> Result, Error> { + // direct lookup not supported by bitcoin core + Ok(None) + } } /// Extension trait for transaction, adding methods to help to match the Transaction to Replace/Redeem requests diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 794aa4825..e1e29e4f9 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -8,7 +8,7 @@ pub use error::Error; use async_trait::async_trait; use backoff::future::retry; use futures::future::{join_all, try_join, try_join_all}; -use std::{sync::Arc, time::Duration}; +use std::{convert::TryFrom, sync::Arc, time::Duration}; use tokio::{sync::Mutex, time::sleep}; const RETRY_DURATION: Duration = Duration::from_millis(1000); @@ -25,14 +25,16 @@ pub struct BitcoinLight { impl BitcoinLight { pub fn new(electrs_url: Option, private_key: PrivateKey) -> Result { let network = private_key.network; - log::info!("Using network: {}", network); let electrs_client = ElectrsClient::new(electrs_url, network)?; + let wallet = wallet::Wallet::new(network, electrs_client.clone()); + // store the derivation key so it can be used for change + wallet.put_p2wpkh_key(private_key.inner)?; Ok(Self { private_key, secp_ctx: secp256k1::Secp256k1::new(), - electrs: electrs_client.clone(), + electrs: electrs_client, transaction_creation_lock: Arc::new(Mutex::new(())), - wallet: wallet::Wallet::new(network, electrs_client), + wallet, }) } @@ -45,29 +47,37 @@ impl BitcoinLight { .ok_or(Error::NoChangeAddress) } - async fn create_transaction( + pub async fn fund_and_sign_transaction( &self, recipient: Address, - sat: u64, + unsigned_tx: Transaction, + change_address: Address, fee_rate: SatPerVbyte, - request_id: Option, + prev_txid: Option, ) -> Result { let lock = self.transaction_creation_lock.clone().lock_owned().await; - let unsigned_tx = self.wallet.create_transaction(recipient.clone(), sat, request_id); - let change_address = self.get_change_address()?; - let mut psbt = self .wallet - .fund_transaction(unsigned_tx, change_address, fee_rate.0.saturating_mul(1000)) + .fund_transaction(unsigned_tx, change_address, fee_rate.0.saturating_mul(1000), prev_txid) .await?; self.wallet.sign_transaction(&mut psbt)?; let signed_tx = psbt.extract_tx(); - Ok(LockedTransaction::new(signed_tx, recipient.to_string(), Some(lock))) } - // TODO: hold tx lock until inclusion otherwise electrs may report stale utxos - // need to check when electrs knows that a utxo has been spent + pub async fn create_transaction( + &self, + recipient: Address, + sat: u64, + fee_rate: SatPerVbyte, + request_id: Option, + ) -> Result { + let unsigned_tx = self.wallet.create_transaction(recipient.clone(), sat, request_id); + let change_address = self.get_change_address()?; + self.fund_and_sign_transaction(recipient, unsigned_tx, change_address, fee_rate, None) + .await + } + async fn send_transaction(&self, transaction: LockedTransaction) -> Result { let txid = self.electrs.send_transaction(transaction.transaction).await?; Ok(txid) @@ -76,6 +86,10 @@ impl BitcoinLight { #[async_trait] impl BitcoinCoreApi for BitcoinLight { + fn is_full_node(&self) -> bool { + false + } + fn network(&self) -> Network { self.private_key.network } @@ -135,7 +149,8 @@ impl BitcoinCoreApi for BitcoinLight { async fn get_block_hash(&self, height: u32) -> Result { match self.electrs.get_block_hash(height).await { Ok(block_hash) => Ok(block_hash), - Err(_) => Err(BitcoinError::InvalidBitcoinHeight), + Err(err) if err.is_not_found() => Err(BitcoinError::InvalidBitcoinHeight), + Err(err) => Err(BitcoinError::ElectrsError(err)), } } @@ -215,7 +230,10 @@ impl BitcoinCoreApi for BitcoinLight { }) .await?; - let (proof, raw_tx) = try_join(self.get_proof(txid, &block_hash), self.get_raw_tx(&txid, &block_hash)).await?; + let (proof, raw_tx) = retry(get_exponential_backoff(), || async { + Ok(try_join(self.get_proof(txid, &block_hash), self.get_raw_tx(&txid, &block_hash)).await?) + }) + .await?; Ok(TransactionMetadata { txid, @@ -227,8 +245,34 @@ impl BitcoinCoreApi for BitcoinLight { }) } - async fn bump_fee(&self, _txid: &Txid, _address: Address, _fee_rate: SatPerVbyte) -> Result { - todo!() + async fn bump_fee(&self, txid: &Txid, address: Address, fee_rate: SatPerVbyte) -> Result { + let mut existing_transaction = self.get_transaction(txid, None).await?; + let return_to_self = existing_transaction + .extract_return_to_self_address(&address.payload)? + .map(|(idx, payload)| { + existing_transaction.output.remove(idx); + Address { + payload, + network: self.network(), + } + }); + + // clear the witnesses for fee estimation + existing_transaction + .input + .iter_mut() + .for_each(|txin| txin.witness.clear()); + let tx = self + .fund_and_sign_transaction( + address, + existing_transaction, + return_to_self.unwrap(), + fee_rate, + Some(*txid), + ) + .await?; + let txid = self.send_transaction(tx).await?; + Ok(txid) } async fn create_and_send_transaction( @@ -292,7 +336,7 @@ impl BitcoinCoreApi for BitcoinLight { let prev_tx = self.get_transaction(&input.previous_output.txid, None).await?; let prev_out = prev_tx .output - .get(input.previous_output.vout as usize) + .get(usize::try_from(input.previous_output.vout)?) .ok_or(Error::NoPrevOut)?; Ok::(prev_out.value) })) @@ -303,4 +347,13 @@ impl BitcoinCoreApi for BitcoinLight { let fee_rate = fee.checked_div(vsize).ok_or(BitcoinError::ArithmeticError)?; Ok(SatPerVbyte(fee_rate)) } + + async fn get_tx_for_op_return( + &self, + address: Address, + amount: u128, + data: H256, + ) -> Result, BitcoinError> { + Ok(self.electrs.get_tx_for_op_return(address, amount, data).await?) + } } diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index e6e9f3193..d02a5ef0f 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -1,20 +1,22 @@ -use bitcoincore_rpc::bitcoin::{ - blockdata::{constants::WITNESS_SCALE_FACTOR, transaction::NonStandardSighashType}, - util::sighash::SighashCache, - EcdsaSig, PackedLockTime, PublicKey, Witness, -}; - use super::{electrs::ElectrsClient, error::Error}; use crate::{ + electrs::Utxo, hashes::Hash, json::bitcoin::EcdsaSighashType, opcodes, psbt, psbt::PartiallySignedTransaction, secp256k1::{All, Message, Secp256k1, SecretKey}, - Address, Builder as ScriptBuilder, Network, OutPoint, PrivateKey, Script, Transaction, TxIn, TxOut, VarInt, H256, + Address, Builder as ScriptBuilder, Network, OutPoint, PrivateKey, Script, Transaction, TxIn, TxOut, Txid, VarInt, + H256, +}; +use bitcoincore_rpc::bitcoin::{ + blockdata::{constants::WITNESS_SCALE_FACTOR, transaction::NonStandardSighashType}, + util::sighash::SighashCache, + EcdsaSig, PackedLockTime, PublicKey, Sequence, Witness, }; +use futures::{stream, Stream, StreamExt}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, VecDeque}, sync::{Arc, RwLock}, }; @@ -57,7 +59,10 @@ impl GetSerializeSize for Witness { fn dummy_sign_input(txin: &mut TxIn, public_key: PublicKey) { // create a dummy signature that is a valid DER-encoding let dummy_signature = { - let m_r_len = 32; + // it is possible to "grind the signature" to encode an r-value + // in 32 bytes, but to be safe we use the default of 33 bytes + // which means that we may overestimate fees + let m_r_len = 33; let m_s_len = 32; let mut vch_sig = vec![0; m_r_len + m_s_len + 7]; @@ -128,6 +133,9 @@ impl FeeRate { } } +// https://github.com/bitcoin/bitcoin/blob/db03248070f39d795b64894c312311df33521427/src/policy/policy.h#L55 +const DUST_RELAY_TX_FEE: FeeRate = FeeRate { n_satoshis_per_k: 3000 }; + struct CoinOutput { value: u64, fee: u64, @@ -194,8 +202,73 @@ fn p2wpkh_script_code(script: &Script) -> Script { .into_script() } +// https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L26 +fn get_dust_threshold(tx_out: &TxOut, dust_relay_fee_in: &FeeRate) -> u64 { + let mut n_size = tx_out.get_serialize_size(); + if tx_out.script_pubkey.is_witness_program() { + n_size += 32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR as u64) + 4; + } else { + n_size += 32 + 4 + 1 + 107 + 4; + } + + dust_relay_fee_in.get_fee(n_size) +} + pub type KeyStore = Arc>>; +pub fn available_coins( + electrs: ElectrsClient, + tx_inputs: Vec, + addresses: Vec
, +) -> impl Stream> + Unpin { + struct StreamState { + electrs: E, + utxos: VecDeque, + tx_inputs: VecDeque, + addresses: VecDeque
, + } + + let state = StreamState { + electrs, + utxos: VecDeque::new(), + tx_inputs: VecDeque::from(tx_inputs), + addresses: VecDeque::from(addresses), + }; + + Box::pin( + stream::unfold(state, move |mut state| async move { + match state.utxos.pop_front() { + Some(utxo) => Some((Ok(utxo), state)), + None => { + if let Some(txin) = state.tx_inputs.pop_front() { + match state.electrs.get_prev_value(&txin.previous_output).await { + Ok(value) => Some(( + Ok(Utxo { + outpoint: txin.previous_output, + value, + }), + state, + )), + Err(e) => Some((Err(Error::ElectrsError(e)), state)), + } + } else if let Some(address) = state.addresses.pop_front() { + match state.electrs.get_utxos_for_address(&address).await { + Ok(utxos) => { + state.utxos = VecDeque::from(utxos); + state.utxos.pop_front().map(|utxo| (Ok(utxo), state)) + } + Err(e) => Some((Err(Error::ElectrsError(e)), state)), + } + } else { + None + } + } + } + }) + .fuse(), + ) +} + #[derive(Clone)] pub struct Wallet { secp: Secp256k1, @@ -230,20 +303,33 @@ impl Wallet { tx: Transaction, change_address: Address, n_satoshis_per_k: u64, + prev_txid: Option, ) -> Result { let recipients_sum = tx.output.iter().map(|tx_out| tx_out.value).sum::(); let m_effective_feerate = FeeRate { n_satoshis_per_k }; - // TODO: calculate actual minimum - let min_viable_change = 0; - let change_output_size = TxOut { + let change_prototype_txout = TxOut { value: 0, script_pubkey: change_address.script_pubkey(), - } - .get_serialize_size(); + }; + let change_output_size = change_prototype_txout.get_serialize_size(); let change_fee = m_effective_feerate.get_fee(change_output_size); + // https://github.com/bitcoin/bitcoin/blob/db03248070f39d795b64894c312311df33521427/src/policy/policy.h#L55 + let m_discard_feerate = DUST_RELAY_TX_FEE; + let dust = get_dust_threshold(&change_prototype_txout, &m_discard_feerate); + let change_spend_size = calculate_maximum_signed_input_size( + OutPoint { + txid: Txid::all_zeros(), + vout: 0, + }, + self.get_pub_key(&change_address.script_pubkey()) + .expect("wallet has key"), + ); + let change_spend_fee = m_discard_feerate.get_fee(change_spend_size); + let min_viable_change = std::cmp::max(change_spend_fee + 1, dust); + let tx_noinputs_size = 10 + VarInt(tx.output.len() as u64).len() as u64 + tx.output.iter().map(|tx_out| tx_out.get_serialize_size()).sum::(); @@ -252,73 +338,78 @@ impl Wallet { // https://github.com/bitcoin/bitcoin/blob/01e1627e25bc5477c40f51da03c3c31b609a85c9/src/wallet/spend.cpp#L896 let selection_target = recipients_sum + not_input_fees; let mut value_to_select = selection_target; - - let mut psbt = PartiallySignedTransaction::from_unsigned_tx(tx)?; let mut select_coins = SelectCoins::new(selection_target); + // not empty if we are fee bumping, include these inputs first + let prev_inputs = tx.input.clone(); + let mut tx_noinputs = tx.clone(); + tx_noinputs.input.clear(); + let mut psbt = PartiallySignedTransaction::from_unsigned_tx(tx_noinputs)?; + // get available coins let addresses = self.key_store.read()?.keys().cloned().collect::>(); - for address in addresses { - log::info!("Found address: {}", address); - // get utxos for address - let utxos = self.electrs.get_utxos_for_address(address).await?; - // TODO: stream this, no need to fetch - for utxo in utxos { - log::info!("Found utxo: {}", utxo.outpoint.txid); - - let script_pubkey = self.electrs.get_script_pubkey(utxo.outpoint).await?; - let public_key = self.get_pub_key(&script_pubkey).expect("wallet has key"); - let input_bytes = calculate_maximum_signed_input_size(utxo.outpoint, public_key); - let coin_output = CoinOutput { + let mut utxo_stream = available_coins(self.electrs.clone(), prev_inputs, addresses); + while let Some(Ok(utxo)) = utxo_stream.next().await { + log::info!("Found utxo: {}", utxo.outpoint.txid); + + if prev_txid.contains(&utxo.outpoint.txid) { + // skip if trying to spend from the tx we are replacing + continue; + } + + let script_pubkey = self.electrs.get_script_pubkey(utxo.outpoint).await?; + let public_key = self.get_pub_key(&script_pubkey).expect("wallet has key"); + let input_bytes = calculate_maximum_signed_input_size(utxo.outpoint, public_key); + let coin_output = CoinOutput { + value: utxo.value, + fee: m_effective_feerate.get_fee(input_bytes), + }; + + let effective_value = coin_output.get_effective_value(); + select_coins.add(coin_output); + value_to_select = value_to_select.saturating_sub(effective_value); + + psbt.unsigned_tx.input.push(TxIn { + previous_output: utxo.outpoint, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, + ..Default::default() + }); + + psbt.inputs.push(psbt::Input { + witness_utxo: Some(TxOut { value: utxo.value, - fee: m_effective_feerate.get_fee(input_bytes), - }; - - let effective_value = coin_output.get_effective_value(); - select_coins.add(coin_output); - value_to_select = value_to_select.saturating_sub(effective_value); - - psbt.unsigned_tx.input.push(TxIn { - previous_output: utxo.outpoint, - ..Default::default() - }); - - psbt.inputs.push(psbt::Input { - witness_utxo: Some(TxOut { - value: utxo.value, - script_pubkey, - }), - ..Default::default() - }); - - if value_to_select == 0 { - // add change output before computing maximum size - let change_amount = select_coins.get_change(min_viable_change, change_fee); - let mut n_change_pos_in_out = None; - if change_amount > 0 { - n_change_pos_in_out = Some(psbt.unsigned_tx.output.len()); - // add change output - psbt.unsigned_tx.output.push(TxOut { - value: change_amount, - script_pubkey: change_address.script_pubkey(), - }); - } + script_pubkey, + }), + ..Default::default() + }); + + if value_to_select == 0 { + // add change output before computing maximum size + let change_amount = select_coins.get_change(min_viable_change, change_fee); + let mut n_change_pos_in_out = None; + if change_amount > 0 { + n_change_pos_in_out = Some(psbt.unsigned_tx.output.len()); + // add change output + psbt.unsigned_tx.output.push(TxOut { + value: change_amount, + script_pubkey: change_address.script_pubkey(), + }); + } - // https://github.com/bitcoin/bitcoin/blob/01e1627e25bc5477c40f51da03c3c31b609a85c9/src/wallet/spend.cpp#L945 - let n_bytes = calculate_maximum_signed_tx_size(&psbt, self); - let fee_needed = m_effective_feerate.get_fee(n_bytes); - let n_fee_ret = select_coins.get_selected_value() - recipients_sum - change_amount; + // https://github.com/bitcoin/bitcoin/blob/01e1627e25bc5477c40f51da03c3c31b609a85c9/src/wallet/spend.cpp#L945 + let n_bytes = calculate_maximum_signed_tx_size(&psbt, self); + let fee_needed = m_effective_feerate.get_fee(n_bytes); + let n_fee_ret = select_coins.get_selected_value() - recipients_sum - change_amount; - if let Some(change_pos) = n_change_pos_in_out { - if fee_needed < n_fee_ret { - log::info!("Fee needed is less than expected"); - let mut change_output = &mut psbt.unsigned_tx.output[change_pos]; - change_output.value += n_fee_ret - fee_needed; - } + if let Some(change_pos) = n_change_pos_in_out { + if fee_needed < n_fee_ret { + log::info!("Fee needed is less than expected"); + let mut change_output = &mut psbt.unsigned_tx.output[change_pos]; + change_output.value += n_fee_ret - fee_needed; } - - return Ok(psbt); } + + return Ok(psbt); } } @@ -358,22 +449,21 @@ impl Wallet { } pub fn sign_transaction(&self, psbt: &mut PartiallySignedTransaction) -> Result<(), Error> { - for inp in 0..psbt.inputs.len() { - let psbt_input = &psbt.inputs[inp]; - + for (index, psbt_input) in psbt.inputs.iter_mut().enumerate() { let prev_out = psbt_input .witness_utxo .clone() .expect("utxo is always set in fund_transaction; qed"); - // Note: we don't support SchnorrSighashType + // NOTE: we don't support SchnorrSighashType let sighash_ty = psbt_input .sighash_type .unwrap_or_else(|| EcdsaSighashType::All.into()) .ecdsa_hash_ty() .map_err(|NonStandardSighashType(ty)| Error::PsbtError(psbt::Error::NonStandardSighashType(ty)))?; - // TODO: support signing p2sh, p2pkh, p2wsh + // NOTE: we don't support signing p2sh, p2pkh, p2wsh inputs + // since the Vault is assumed to only receive p2wpkh payments let script_code = if prev_out.script_pubkey.is_v0_p2wpkh() { Ok(p2wpkh_script_code(&prev_out.script_pubkey)) } else { @@ -381,10 +471,9 @@ impl Wallet { }?; let mut sig_hasher = SighashCache::new(&psbt.unsigned_tx); - let sig_hash = sig_hasher.segwit_signature_hash(inp, &script_code, prev_out.value, sighash_ty)?; + let sig_hash = sig_hasher.segwit_signature_hash(index, &script_code, prev_out.value, sighash_ty)?; let private_key = self.get_priv_key(&prev_out.script_pubkey)?; - let sig = self .secp .sign_ecdsa(&Message::from_slice(&sig_hash.into_inner()[..])?, &private_key.inner); @@ -394,16 +483,11 @@ impl Wallet { hash_ty: sighash_ty, }; - // TODO: can we write directly to final_script_witness here? - psbt.inputs[inp] - .partial_sigs - .insert(private_key.public_key(&self.secp), final_signature); - } - - for psbt_input in psbt.inputs.iter_mut() { - let (key, sig) = psbt_input.partial_sigs.iter().next().expect("signature set above; qed"); // https://github.com/bitcoin/bitcoin/blob/607d5a46aa0f5053d8643a3e2c31a69bfdeb6e9f/src/script/sign.cpp#L125 - psbt_input.final_script_witness = Some(Witness::from_vec(vec![sig.to_vec(), key.to_bytes()])); + psbt_input.final_script_witness = Some(Witness::from_vec(vec![ + final_signature.to_vec(), + private_key.public_key(&self.secp).to_bytes(), + ])); } Ok(()) @@ -472,6 +556,26 @@ mod tests { Ok(()) } + #[test] + fn should_calculate_signed_size() -> Result<(), Box> { + let signed_tx_bytes = Vec::from_hex( + "02000000000102fc0894755eb329675f4ced4c4aef159c6215e0a4e127f9a0dfabe0cae23fd512000000\ + 0000ffffffff7504b0108726fb75b26151db2c2a6ae9aecfe2809f2e166c017e3f0eca1fcbec000000000\ + 0ffffffff030c0e000000000000160014aa8ef374cafadfca76902ddb5cf61c60bbfd9d85000000000000\ + 0000226a2014ca9d53b116b5597fa10b913fad4817b0fbbe0a907d2fc9b3dbd4747b93c17ff7140000000\ + 000001600144e1be54fdbfb20928aef4c378fb9ca6afaea2bed02483045022100e3750e734e674c7395f7\ + 7287afe1446552193bfdc79ce3b282b1c554d20a1517022047c8cbbf52d73cea3d5c930e0d9412fd1d5fa\ + 8c9e4a91a3f705495df8204f788012103a782f4e40dbaa8f09ca3c94993b65dc82028e9b17ca9a8a42c02\ + 11e630e3464f02473044022061fd6cfc6d7b323b91402d9a4c143901cd3a8b2c68977122596f99f44d5e0\ + 27e02207407b04ac4f39edd0b0a21ef28bac789113d801e92c8d0e23b5e2b4c9f3e611101210326084ec5\ + 72920e3a37eb2b254f42cfcf162e3e9eebc776c13642805c2985500800000000", + ) + .unwrap(); + let tx: Transaction = deserialize(&signed_tx_bytes)?; + assert_eq!(get_virtual_transaction_size(tx.weight() as u64), 252); + Ok(()) + } + #[tokio::test(flavor = "multi_thread")] async fn test_calculate_fees() -> Result<(), Box> { let tx = Transaction { diff --git a/bitcoin/tests/electrs.rs b/bitcoin/tests/electrs.rs new file mode 100644 index 000000000..b7f31463b --- /dev/null +++ b/bitcoin/tests/electrs.rs @@ -0,0 +1,217 @@ +#![cfg(feature = "uses-bitcoind")] + +use bitcoin::{ + secp256k1::{constants::SECRET_KEY_SIZE, Secp256k1}, + Address, AddressType, Amount, Auth, BitcoinCoreApi, BitcoinLight, BlockHash, Client, ElectrsClient, Error, Hash, + Network, PrivateKey, PublicKey, RpcApi, SatPerVbyte, SecretKey, H256, +}; +use futures::{future::join, Future}; +use rand::{thread_rng, Rng}; +use serial_test::serial; +use std::{env::var, time::Duration}; +use tokio::time::{sleep, timeout}; + +const DEFAULT_NETWORK: Network = Network::Regtest; + +fn new_random_key_pair() -> (PrivateKey, PublicKey) { + // NOTE: private key wif cannot encode regtest + let raw_secret_key: [u8; SECRET_KEY_SIZE] = thread_rng().gen(); + let secret_key = SecretKey::from_slice(&raw_secret_key).unwrap(); + let private_key = PrivateKey::new(secret_key, DEFAULT_NETWORK); + let public_key = PublicKey::from_private_key(&Secp256k1::new(), &private_key); + (private_key, public_key) +} + +fn new_bitcoin_client() -> Client { + Client::new( + &var("BITCOIN_RPC_URL").expect("BITCOIN_RPC_URL not set"), + Auth::UserPass( + var("BITCOIN_RPC_USER").expect("BITCOIN_RPC_USER not set"), + var("BITCOIN_RPC_PASS").expect("BITCOIN_RPC_PASS not set"), + ), + ) + .unwrap() +} + +fn new_bitcoin_light() -> BitcoinLight { + BitcoinLight::new( + Some(var("ELECTRS_URL").expect("ELECTRS_URL not set")), + new_random_key_pair().0, + ) + .unwrap() +} + +fn new_electrs() -> ElectrsClient { + ElectrsClient::new(Some(var("ELECTRS_URL").expect("ELECTRS_URL not set")), DEFAULT_NETWORK).unwrap() +} + +async fn wait_for_success(f: F) -> T +where + F: Fn() -> R, + R: Future>, +{ + timeout(Duration::from_secs(20), async move { + loop { + match f().await { + Ok(x) => return x, + Err(_) => sleep(Duration::from_millis(10)).await, + } + } + }) + .await + .expect("Time limit elapsed") +} + +fn mine_blocks(block_num: u64, maybe_address: Option
) -> BlockHash { + let bitcoin_client = new_bitcoin_client(); + let address = + maybe_address.unwrap_or_else(|| bitcoin_client.get_new_address(None, Some(AddressType::Bech32)).unwrap()); + bitcoin_client + .generate_to_address(block_num, &address) + .unwrap() + .last() + .unwrap() + .clone() +} + +async fn fund_wallet(bitcoin_light: &BitcoinLight) -> Result<(), Error> { + // need at least 100 confirmations otherwise we get + // this error: bad-txns-premature-spend-of-coinbase + let public_key = new_random_key_pair().1; + let address = Address::p2wpkh(&public_key, DEFAULT_NETWORK).unwrap(); + mine_blocks(101, Some(address)); + + // fund the master key + let master_public_key = bitcoin_light.get_new_public_key().await?; + let master_address = Address::p2wpkh(&master_public_key, DEFAULT_NETWORK).unwrap(); + // electrs may still include unconfirmed coinbase txs so to + // avoid this we mine and then send it to the master address + new_bitcoin_client().send_to_address( + &master_address, + Amount::from_sat(100000), + None, + None, + None, + None, + None, + None, + )?; + mine_blocks(1, None); + + // wait for electrs to pickup utxo + wait_for_success(|| async { + new_electrs() + .get_utxos_for_address(&master_address) + .await + .map_err(|_| ())? + .len() + .gt(&0) + .then_some(()) + .ok_or(()) + }) + .await; + Ok(()) +} + +#[tokio::test] +#[serial] +async fn should_create_transactions() -> Result<(), Error> { + let bitcoin_light = new_bitcoin_light(); + fund_wallet(&bitcoin_light).await?; + + let address1 = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); + let address2 = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); + let (res1, res2) = join( + bitcoin_light.create_and_send_transaction( + address1.clone(), + 1000, + SatPerVbyte(1), + Some(H256::from_slice(&[1; 32])), + ), + bitcoin_light.create_and_send_transaction( + address2.clone(), + 1000, + SatPerVbyte(1), + Some(H256::from_slice(&[2; 32])), + ), + ) + .await; + + let txid1 = res1?; + let txid2 = res2?; + + let mempool_txs: Vec<_> = bitcoin_light.get_mempool_transactions().await?.collect(); + assert_eq!(mempool_txs.len(), 2); + assert!(bitcoin_light.is_in_mempool(txid1).await?, "Txid1 not in mempool"); + assert!(bitcoin_light.is_in_mempool(txid2).await?, "Txid2 not in mempool"); + + // mine a block to include our transactions + let block_hash = mine_blocks(1, None); + // wait for electrs to pickup block + wait_for_success(|| async { bitcoin_light.get_block(&block_hash).await }).await; + + assert!( + bitcoin_light.get_proof(txid1, &BlockHash::all_zeros()).await.is_ok(), + "Txid1 not confirmed" + ); + assert!( + bitcoin_light.get_proof(txid2, &BlockHash::all_zeros()).await.is_ok(), + "Txid2 not confirmed" + ); + + assert!(bitcoin_light + .get_tx_for_op_return(address1, 1000, H256::from_slice(&[1; 32])) + .await? + .is_some()); + + assert!(bitcoin_light + .get_tx_for_op_return(address2, 1000, H256::from_slice(&[2; 32])) + .await? + .is_some()); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn should_bump_fee() -> Result<(), Error> { + let bitcoin_light = new_bitcoin_light(); + fund_wallet(&bitcoin_light).await?; + + let address = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); + let txid1 = bitcoin_light + .create_and_send_transaction(address.clone(), 1000, SatPerVbyte(1), Some(H256::from_slice(&[1; 32]))) + .await?; + assert_eq!(SatPerVbyte(1), bitcoin_light.fee_rate(txid1).await?); + + let txid2 = bitcoin_light.bump_fee(&txid1, address, SatPerVbyte(2)).await?; + assert_eq!(SatPerVbyte(2), bitcoin_light.fee_rate(txid2).await?); + + let block_hash = mine_blocks(1, None); + wait_for_success(|| async { new_electrs().get_block_header(&block_hash).await }).await; + + assert!( + bitcoin_light.get_proof(txid1, &BlockHash::all_zeros()).await.is_err(), + "Txid1 should not exist" + ); + assert!( + bitcoin_light.get_proof(txid2, &BlockHash::all_zeros()).await.is_ok(), + "Txid2 should be confirmed" + ); + + Ok(()) +} + +#[tokio::test] +#[serial] +async fn should_page_address_history() -> Result<(), Error> { + let public_key = new_random_key_pair().1; + let address = Address::p2wpkh(&public_key, DEFAULT_NETWORK).unwrap(); + let block_hash = mine_blocks(100, Some(address.clone())); + wait_for_success(|| async { new_electrs().get_block_header(&block_hash).await }).await; + + let txs = new_electrs().get_address_tx_history_full(&address.to_string()).await?; + assert!(txs.len().eq(&100)); + + Ok(()) +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..d4b66012a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +version: "3.8" +services: + bitcoind: + image: "ruimarinho/bitcoin-core:22" + command: + - -regtest + - -server + - -rpcbind=0.0.0.0 + - -rpcallowip=0.0.0.0/0 + - -rpcuser=rpcuser + - -rpcpassword=rpcpassword + - -fallbackfee=0.0002 + ports: + - "18443:18443" + bitcoin-cli: + image: "ruimarinho/bitcoin-core:22" + command: + - /bin/sh + - -c + - | + bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword createwallet Alice + ALICE_ADDRESS=$$(bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword -rpcwallet=Alice getnewaddress) + # coins need 100 confirmations to be spendable + bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword generatetoaddress 101 $${ALICE_ADDRESS} + electrs: + image: "interlayhq/electrs:latest" + command: + - electrs + - -vvvv + - --network + - regtest + - --jsonrpc-import + - --cors + - "*" + - --cookie + - "rpcuser:rpcpassword" + - --daemon-rpc-addr + - "bitcoind:18443" + - --http-addr + - "[::0]:3002" + - --index-unspendables + ports: + - "3002:3002" + restart: always diff --git a/runtime/src/integration/bitcoin_simulator.rs b/runtime/src/integration/bitcoin_simulator.rs index 3cce14239..7afebece4 100644 --- a/runtime/src/integration/bitcoin_simulator.rs +++ b/runtime/src/integration/bitcoin_simulator.rs @@ -508,4 +508,13 @@ impl BitcoinCoreApi for MockBitcoinCore { async fn fee_rate(&self, txid: Txid) -> Result { unimplemented!() } + + async fn get_tx_for_op_return( + &self, + _address: Address, + _amount: u128, + _data: H256, + ) -> Result, BitcoinError> { + Ok(None) + } } diff --git a/vault/src/execution.rs b/vault/src/execution.rs index a28292869..1c97213d3 100644 --- a/vault/src/execution.rs +++ b/vault/src/execution.rs @@ -285,7 +285,7 @@ impl Request { .filter_map(|x| async { match btc_rpc.is_in_mempool(txid_copy).await { Ok(false) => { - // if not in mempool anymore, don't propagate the event (even if it is an error) + // if not in mempool anymore, don't propagate the event (even if it is an error) tracing::debug!("Txid not in mempool anymore..."); None } @@ -418,6 +418,48 @@ impl Request { } } +fn create_payment_worker( + shutdown_tx: ShutdownSender, + parachain_rpc: InterBtcParachain, + vault_id_manager: VaultIdManager, + request: Request, + txid: Txid, + num_confirmations: u32, + auto_rbf: bool, +) { + tracing::info!( + "{:?} request #{:?} has valid bitcoin payment - processing...", + request.request_type, + request.hash + ); + spawn_cancelable(shutdown_tx.subscribe(), async move { + let btc_rpc = match vault_id_manager.get_bitcoin_rpc(&request.vault_id).await { + Some(x) => x, + None => { + tracing::error!( + "Failed to fetch bitcoin rpc for vault {}", + request.vault_id.pretty_print() + ); + return; // nothing we can do - bail + } + }; + + match request + .wait_for_inclusion(¶chain_rpc, &btc_rpc, num_confirmations, txid, auto_rbf) + .await + { + Ok(tx_metadata) => { + if let Err(e) = request.execute(parachain_rpc.clone(), tx_metadata).await { + tracing::error!("Failed to execute request #{}: {}", request.hash, e); + } + } + Err(e) => { + tracing::error!("Error while waiting for inclusion for request #{}: {}", request.hash, e); + } + } + }); +} + /// Queries the parachain for open requests and executes them. It checks the /// bitcoin blockchain to see if a payment has already been made. #[allow(clippy::too_many_arguments)] @@ -465,82 +507,94 @@ pub async fn execute_open_requests( None => return Ok(()), // the iterator is empty so we have nothing to do }; - let rate_limiter = RateLimiter::direct(YIELD_RATE); + // NOTE: block iteration is expensive so only use a full node + // direct lookup is possible with the light client + if read_only_btc_rpc.is_full_node() { + let rate_limiter = RateLimiter::direct(YIELD_RATE); + // iterate through transactions in reverse order, starting from those in the mempool, and + // gracefully fail on encountering a pruned blockchain + let mut transaction_stream = bitcoin::reverse_stream_transactions(&read_only_btc_rpc, btc_start_height).await?; + while let Some(result) = transaction_stream.next().await { + if rate_limiter.check().is_ok() { + // give the outer `select` a chance to check the shutdown signal + tokio::task::yield_now().await; + } + + // When there is an error, we have to make a choice. Either we restart or + // continue processing. Both options have their risks: if we restart, it's + // possible that the vault will never manage to start up, causing us to + // fail to process requests. On the other hand, if we ignore transactions, + // we risk double paying. We choose to restart only for network errors, + // since these are not expected to be persistent. Other errors could be + // persistent, so we keep going. + let tx = match result { + Ok(x) => x, + Err(e) if e.is_transport_error() => { + return Err(e.into()); + } + Err(e) => { + tracing::warn!("Failed to process transaction: {}", e); + continue; + } + }; - // iterate through transactions in reverse order, starting from those in the mempool, and - // gracefully fail on encountering a pruned blockchain - let mut transaction_stream = bitcoin::reverse_stream_transactions(&read_only_btc_rpc, btc_start_height).await?; - while let Some(result) = transaction_stream.next().await { - if rate_limiter.check().is_ok() { - // give the outer `select` a chance to check the shutdown signal - tokio::task::yield_now().await; + // get the request this transaction corresponds to, if any + if let Some(request) = get_request_for_btc_tx(&tx, &open_requests) { + // remove request from the hashmap + open_requests.retain(|&key, _| key != request.hash); + + // start a new task to (potentially) await confirmation and to execute on the parachain + // make copies of the variables we move into the task + create_payment_worker( + shutdown_tx.clone(), + parachain_rpc.clone(), + vault_id_manager.clone(), + request, + tx.txid(), + num_confirmations, + auto_rbf, + ); + } } + } - // When there is an error, we have to make a choice. Either we restart or - // continue processing. Both options have their risks: if we restart, it's - // possible that the vault will never manage to start up, causing us to - // fail to process requests. On the other hand, if we ignore transactions, - // we risk double paying. We choose to restart only for network errors, - // since these are not expected to be persistent. Other errors could be - // persistent, so we keep going. - let tx = match result { - Ok(x) => x, - Err(e) if e.is_transport_error() => { - return Err(e.into()); + // All requests remaining in the hashmap did not have a bitcoin payment yet + // or were not found in the stream, check if we can fetch by id directly + // or just pay and execute all requests individually + for (id, request) in open_requests { + // try finding transaction directly (if supported) + match read_only_btc_rpc + .get_tx_for_op_return( + request + .btc_address + .to_address(read_only_btc_rpc.network()) + .map_err(BitcoinError::ConversionError)?, + request.amount, + id, + ) + .await + { + Ok(Some(txid)) => { + create_payment_worker( + shutdown_tx.clone(), + parachain_rpc.clone(), + vault_id_manager.clone(), + request, + txid, + num_confirmations, + auto_rbf, + ); + // task will handling execution + continue; } - Err(e) => { - tracing::warn!("Failed to process transaction: {}", e); + Ok(None) => {} // make payment + Err(err) => { + tracing::error!("Failed to fetch tx for OP_RETURN {}", err); + // TODO: can we handle this error and still make the payment? continue; } - }; - - // get the request this transaction corresponds to, if any - if let Some(request) = get_request_for_btc_tx(&tx, &open_requests) { - // remove request from the hashmap - open_requests.retain(|&key, _| key != request.hash); - - tracing::info!( - "{:?} request #{:?} has valid bitcoin payment - processing...", - request.request_type, - request.hash - ); - - // start a new task to (potentially) await confirmation and to execute on the parachain - // make copies of the variables we move into the task - let parachain_rpc = parachain_rpc.clone(); - let btc_rpc = vault_id_manager.clone(); - spawn_cancelable(shutdown_tx.subscribe(), async move { - let btc_rpc = match btc_rpc.get_bitcoin_rpc(&request.vault_id).await { - Some(x) => x, - None => { - tracing::error!( - "Failed to fetch bitcoin rpc for vault {}", - request.vault_id.pretty_print() - ); - return; // nothing we can do - bail - } - }; - - match request - .wait_for_inclusion(¶chain_rpc, &btc_rpc, num_confirmations, tx.txid(), auto_rbf) - .await - { - Ok(tx_metadata) => { - if let Err(e) = request.execute(parachain_rpc.clone(), tx_metadata).await { - tracing::error!("Failed to execute request #{}: {}", request.hash, e); - } - } - Err(e) => { - tracing::error!("Error while waiting for inclusion for request #{}: {}", request.hash, e); - } - } - }); } - } - // All requests remaining in the hashmap did not have a bitcoin payment yet, so pay - // and execute all of these - for (_, request) in open_requests { // there are potentially a large number of open requests - pay and execute each // in a separate task to ensure that awaiting confirmations does not significantly // delay other requests @@ -734,6 +788,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; @@ -768,6 +823,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, BitcoinError>; } } diff --git a/vault/src/issue.rs b/vault/src/issue.rs index a9207f6d9..73203c7b6 100644 --- a/vault/src/issue.rs +++ b/vault/src/issue.rs @@ -44,6 +44,8 @@ pub async fn process_issue_requests( num_confirmations: u32, random_delay: Arc>, ) -> Result<(), ServiceError> { + // NOTE: we should not stream transactions if using the light client + // since it is quite expensive to fetch all transactions per block let mut stream = bitcoin::stream_in_chain_transactions(bitcoin_core.clone(), btc_start_height, num_confirmations).await; diff --git a/vault/src/metrics.rs b/vault/src/metrics.rs index 0be4739e0..2840874aa 100644 --- a/vault/src/metrics.rs +++ b/vault/src/metrics.rs @@ -790,6 +790,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; @@ -824,6 +825,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, BitcoinError>; } } diff --git a/vault/src/replace.rs b/vault/src/replace.rs index a3f1b7c9d..816d664ce 100644 --- a/vault/src/replace.rs +++ b/vault/src/replace.rs @@ -233,6 +233,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; @@ -290,6 +291,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, BitcoinError>; } } diff --git a/vault/src/system.rs b/vault/src/system.rs index b4480ffe1..64cf2bffa 100644 --- a/vault/src/system.rs +++ b/vault/src/system.rs @@ -750,7 +750,7 @@ impl VaultService { ( "Issue Executor", maybe_run( - !self.config.no_issue_execution, + !self.config.no_issue_execution && self.btc_rpc_master_wallet.is_full_node(), issue::process_issue_requests( self.btc_rpc_master_wallet.clone(), self.btc_parachain.clone(),