From 4a577357361bd78af336a4d7a28a367cb3bdd6d3 Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Tue, 21 Mar 2023 14:14:53 +0000 Subject: [PATCH 01/13] refactor: vault light client improvements Signed-off-by: Gregory Hill --- Cargo.lock | 10 +- bitcoin/src/electrs/mod.rs | 29 ++---- bitcoin/src/iter.rs | 6 +- bitcoin/src/lib.rs | 11 +++ bitcoin/src/light/mod.rs | 18 +++- bitcoin/src/light/wallet.rs | 87 ++++++++++++----- docker-compose.yml | 44 +++++++++ vault/src/execution.rs | 183 ++++++++++++++++++++++-------------- vault/src/issue.rs | 2 + vault/src/metrics.rs | 1 + vault/src/replace.rs | 1 + vault/src/system.rs | 2 +- 12 files changed, 270 insertions(+), 124 deletions(-) create mode 100644 docker-compose.yml diff --git a/Cargo.lock b/Cargo.lock index 67dced928..c28defba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15984,12 +15984,12 @@ dependencies = [ "pkg-config", ] -[[patch.unused]] -name = "sp-serializer" -version = "4.0.0-dev" -source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" - [[patch.unused]] name = "orml-xcm" version = "0.4.1-dev" source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" + +[[patch.unused]] +name = "sp-serializer" +version = "4.0.0-dev" +source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index c9e46f82f..8c5b4fc6e 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -24,13 +24,6 @@ pub struct Utxo { pub value: u64, } -#[allow(dead_code)] -pub struct TxData { - pub txid: Txid, - pub raw_merkle_proof: Vec, - pub raw_tx: Vec, -} - #[derive(Debug)] pub struct TxInfo { pub confirmations: u32, @@ -125,6 +118,7 @@ impl ElectrsClient { Ok(deserialize(&raw_block_header)?) } + // TODO: this is expensive and not strictly required by the light-client, deprecate? pub(crate) async fn get_transactions_in_block(&self, hash: &BlockHash) -> Result, Error> { let raw_txids: Vec = self.get_and_decode(&format!("/block/{hash}/txids")).await?; let txids: Vec = raw_txids @@ -220,8 +214,7 @@ impl ElectrsClient { Ok(Txid::from_str(&txid)?) } - #[allow(dead_code)] - pub(crate) async fn get_tx_by_op_return(&self, data: H256) -> Result, Error> { + pub(crate) async fn get_tx_for_op_return(&self, data: H256) -> Result, Error> { let script = ScriptBuilder::new() .push_opcode(opcodes::OP_RETURN) .push_slice(data.as_bytes()) @@ -234,6 +227,7 @@ impl ElectrsClient { }; // TODO: page this using last_seen_txid + // NOTE: includes unconfirmed txs let txs: Vec = self .get_and_decode(&format!( "/scripthash/{scripthash}/txs", @@ -242,22 +236,11 @@ impl ElectrsClient { .await?; log::info!("Found {} transactions", txs.len()); - // for now, use the first tx - should probably return - // an error if there are more that one + // TODO: check payment amount or throw error + // if there are multiple transactions if let Some(tx) = txs.first().cloned() { let txid = Txid::from_str(&tx.txid)?; - log::info!("Fetching merkle proof"); - // TODO: return error if not confirmed - let raw_merkle_proof = self.get_raw_tx_merkle_proof(&txid).await?; - - log::info!("Fetching transaction"); - let raw_tx = self.get_raw_tx(&txid).await?; - - Ok(Some(TxData { - txid, - raw_merkle_proof, - raw_tx, - })) + Ok(Some(txid)) } else { Ok(None) } diff --git a/bitcoin/src/iter.rs b/bitcoin/src/iter.rs index a83be3fd2..d49d86023 100644 --- a/bitcoin/src/iter.rs +++ b/bitcoin/src/iter.rs @@ -35,7 +35,7 @@ pub async fn reverse_stream_transactions( /// * `stop_height` - height of the last block the iterator will return transactions from /// * `stop_at_pruned` - whether to gracefully stop if a pruned blockchain is encountered; /// otherwise, will throw an error -pub async fn reverse_stream_in_chain_transactions( +async fn reverse_stream_in_chain_transactions( rpc: &DynBitcoinCoreApi, stop_height: u32, ) -> impl Stream> + Send + Unpin + '_ { @@ -62,7 +62,7 @@ pub async fn reverse_stream_in_chain_transactions( /// * `stop_height` - height of the last block the stream will return /// * `stop_at_pruned` - whether to gracefully stop if a pruned blockchain is encountered; /// otherwise, will throw an error -pub async fn reverse_stream_blocks( +async fn reverse_stream_blocks( rpc: &DynBitcoinCoreApi, stop_height: u32, ) -> impl Stream> + Unpin + '_ { @@ -202,6 +202,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; @@ -262,6 +263,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, data: H256) -> Result, Error>; } } diff --git a/bitcoin/src/lib.rs b/bitcoin/src/lib.rs index 319d99b34..e0cd067d1 100644 --- a/bitcoin/src/lib.rs +++ b/bitcoin/src/lib.rs @@ -124,6 +124,10 @@ pub struct TransactionMetadata { #[async_trait] pub trait BitcoinCoreApi { + fn is_full_node(&self) -> bool { + true + } + fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; @@ -200,6 +204,8 @@ pub trait BitcoinCoreApi { async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + + async fn get_tx_for_op_return(&self, data: H256) -> Result, Error>; } struct LockedTransaction { @@ -1070,6 +1076,11 @@ impl BitcoinCoreApi for BitcoinCore { let fee_rate = fee.checked_div(vsize).ok_or(Error::ArithmeticError)?; Ok(SatPerVbyte(fee_rate.try_into()?)) } + + async fn get_tx_for_op_return(&self, _data: H256) -> Result, Error> { + // direct lookup not supported by bitcoin core + Ok(None) + } } /// Extension trait for transaction, adding methods to help to match the Transaction to Replace/Redeem requests diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 794aa4825..340a3d246 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -76,6 +76,10 @@ impl BitcoinLight { #[async_trait] impl BitcoinCoreApi for BitcoinLight { + fn is_full_node(&self) -> bool { + false + } + fn network(&self) -> Network { self.private_key.network } @@ -135,7 +139,10 @@ impl BitcoinCoreApi for BitcoinLight { async fn get_block_hash(&self, height: u32) -> Result { match self.electrs.get_block_hash(height).await { Ok(block_hash) => Ok(block_hash), - Err(_) => Err(BitcoinError::InvalidBitcoinHeight), + Err(_err) => { + // TODO: handle error + Err(BitcoinError::InvalidBitcoinHeight) + } } } @@ -215,7 +222,10 @@ impl BitcoinCoreApi for BitcoinLight { }) .await?; - let (proof, raw_tx) = try_join(self.get_proof(txid, &block_hash), self.get_raw_tx(&txid, &block_hash)).await?; + let (proof, raw_tx) = retry(get_exponential_backoff(), || async { + Ok(try_join(self.get_proof(txid, &block_hash), self.get_raw_tx(&txid, &block_hash)).await?) + }) + .await?; Ok(TransactionMetadata { txid, @@ -303,4 +313,8 @@ impl BitcoinCoreApi for BitcoinLight { let fee_rate = fee.checked_div(vsize).ok_or(BitcoinError::ArithmeticError)?; Ok(SatPerVbyte(fee_rate)) } + + async fn get_tx_for_op_return(&self, data: H256) -> Result, BitcoinError> { + Ok(self.electrs.get_tx_for_op_return(data).await?) + } } diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index e6e9f3193..dc7f4b65a 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -11,7 +11,8 @@ use crate::{ opcodes, psbt, psbt::PartiallySignedTransaction, secp256k1::{All, Message, Secp256k1, SecretKey}, - Address, Builder as ScriptBuilder, Network, OutPoint, PrivateKey, Script, Transaction, TxIn, TxOut, VarInt, H256, + Address, Builder as ScriptBuilder, Network, OutPoint, PrivateKey, Script, Transaction, TxIn, TxOut, Txid, VarInt, + H256, }; use std::{ collections::BTreeMap, @@ -57,7 +58,10 @@ impl GetSerializeSize for Witness { fn dummy_sign_input(txin: &mut TxIn, public_key: PublicKey) { // create a dummy signature that is a valid DER-encoding let dummy_signature = { - let m_r_len = 32; + // it is possible to "grind the signature" to encode an r-value + // in 32 bytes, but to be safe we use the default of 33 bytes + // which means that we may overestimate fees + let m_r_len = 33; let m_s_len = 32; let mut vch_sig = vec![0; m_r_len + m_s_len + 7]; @@ -128,6 +132,9 @@ impl FeeRate { } } +// https://github.com/bitcoin/bitcoin/blob/db03248070f39d795b64894c312311df33521427/src/policy/policy.h#L55 +const DUST_RELAY_TX_FEE: FeeRate = FeeRate { n_satoshis_per_k: 3000 }; + struct CoinOutput { value: u64, fee: u64, @@ -194,6 +201,18 @@ fn p2wpkh_script_code(script: &Script) -> Script { .into_script() } +// https://github.com/bitcoin/bitcoin/blob/e9262ea32a6e1d364fb7974844fadc36f931f8c6/src/policy/policy.cpp#L26 +fn get_dust_threshold(tx_out: &TxOut, dust_relay_fee_in: &FeeRate) -> u64 { + let mut n_size = tx_out.get_serialize_size(); + if tx_out.script_pubkey.is_witness_program() { + n_size += 32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR as u64) + 4; + } else { + n_size += 32 + 4 + 1 + 107 + 4; + } + + return dust_relay_fee_in.get_fee(n_size); +} + pub type KeyStore = Arc>>; #[derive(Clone)] @@ -235,15 +254,27 @@ impl Wallet { let m_effective_feerate = FeeRate { n_satoshis_per_k }; - // TODO: calculate actual minimum - let min_viable_change = 0; - let change_output_size = TxOut { + let change_prototype_txout = TxOut { value: 0, script_pubkey: change_address.script_pubkey(), - } - .get_serialize_size(); + }; + let change_output_size = change_prototype_txout.get_serialize_size(); let change_fee = m_effective_feerate.get_fee(change_output_size); + // https://github.com/bitcoin/bitcoin/blob/db03248070f39d795b64894c312311df33521427/src/policy/policy.h#L55 + let m_discard_feerate = DUST_RELAY_TX_FEE; + let dust = get_dust_threshold(&change_prototype_txout, &m_discard_feerate); + let change_spend_size = calculate_maximum_signed_input_size( + OutPoint { + txid: Txid::all_zeros(), + vout: 0, + }, + self.get_pub_key(&change_address.script_pubkey()) + .expect("wallet has key"), + ); + let change_spend_fee = m_discard_feerate.get_fee(change_spend_size); + let min_viable_change = std::cmp::max(change_spend_fee + 1, dust); + let tx_noinputs_size = 10 + VarInt(tx.output.len() as u64).len() as u64 + tx.output.iter().map(|tx_out| tx_out.get_serialize_size()).sum::(); @@ -358,15 +389,13 @@ impl Wallet { } pub fn sign_transaction(&self, psbt: &mut PartiallySignedTransaction) -> Result<(), Error> { - for inp in 0..psbt.inputs.len() { - let psbt_input = &psbt.inputs[inp]; - + for (index, psbt_input) in psbt.inputs.iter_mut().enumerate() { let prev_out = psbt_input .witness_utxo .clone() .expect("utxo is always set in fund_transaction; qed"); - // Note: we don't support SchnorrSighashType + // NOTE: we don't support SchnorrSighashType let sighash_ty = psbt_input .sighash_type .unwrap_or_else(|| EcdsaSighashType::All.into()) @@ -381,10 +410,9 @@ impl Wallet { }?; let mut sig_hasher = SighashCache::new(&psbt.unsigned_tx); - let sig_hash = sig_hasher.segwit_signature_hash(inp, &script_code, prev_out.value, sighash_ty)?; + let sig_hash = sig_hasher.segwit_signature_hash(index, &script_code, prev_out.value, sighash_ty)?; let private_key = self.get_priv_key(&prev_out.script_pubkey)?; - let sig = self .secp .sign_ecdsa(&Message::from_slice(&sig_hash.into_inner()[..])?, &private_key.inner); @@ -394,16 +422,11 @@ impl Wallet { hash_ty: sighash_ty, }; - // TODO: can we write directly to final_script_witness here? - psbt.inputs[inp] - .partial_sigs - .insert(private_key.public_key(&self.secp), final_signature); - } - - for psbt_input in psbt.inputs.iter_mut() { - let (key, sig) = psbt_input.partial_sigs.iter().next().expect("signature set above; qed"); // https://github.com/bitcoin/bitcoin/blob/607d5a46aa0f5053d8643a3e2c31a69bfdeb6e9f/src/script/sign.cpp#L125 - psbt_input.final_script_witness = Some(Witness::from_vec(vec![sig.to_vec(), key.to_bytes()])); + psbt_input.final_script_witness = Some(Witness::from_vec(vec![ + final_signature.to_vec(), + private_key.public_key(&self.secp).to_bytes(), + ])); } Ok(()) @@ -472,6 +495,26 @@ mod tests { Ok(()) } + #[test] + fn should_calculate_signed_size() -> Result<(), Box> { + let signed_tx_bytes = Vec::from_hex( + "02000000000102fc0894755eb329675f4ced4c4aef159c6215e0a4e127f9a0dfabe0cae23fd512000000\ + 0000ffffffff7504b0108726fb75b26151db2c2a6ae9aecfe2809f2e166c017e3f0eca1fcbec000000000\ + 0ffffffff030c0e000000000000160014aa8ef374cafadfca76902ddb5cf61c60bbfd9d85000000000000\ + 0000226a2014ca9d53b116b5597fa10b913fad4817b0fbbe0a907d2fc9b3dbd4747b93c17ff7140000000\ + 000001600144e1be54fdbfb20928aef4c378fb9ca6afaea2bed02483045022100e3750e734e674c7395f7\ + 7287afe1446552193bfdc79ce3b282b1c554d20a1517022047c8cbbf52d73cea3d5c930e0d9412fd1d5fa\ + 8c9e4a91a3f705495df8204f788012103a782f4e40dbaa8f09ca3c94993b65dc82028e9b17ca9a8a42c02\ + 11e630e3464f02473044022061fd6cfc6d7b323b91402d9a4c143901cd3a8b2c68977122596f99f44d5e0\ + 27e02207407b04ac4f39edd0b0a21ef28bac789113d801e92c8d0e23b5e2b4c9f3e611101210326084ec5\ + 72920e3a37eb2b254f42cfcf162e3e9eebc776c13642805c2985500800000000", + ) + .unwrap(); + let tx: Transaction = deserialize(&signed_tx_bytes)?; + assert_eq!(get_virtual_transaction_size(tx.weight() as u64), 252); + Ok(()) + } + #[tokio::test(flavor = "multi_thread")] async fn test_calculate_fees() -> Result<(), Box> { let tx = Transaction { diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..d50fc3bba --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,44 @@ +version: "3.8" +services: + bitcoind: + image: "ruimarinho/bitcoin-core:22" + command: + - -regtest + - -server + - -rpcbind=0.0.0.0 + - -rpcallowip=0.0.0.0/0 + - -rpcuser=rpcuser + - -rpcpassword=rpcpassword + - -fallbackfee=0.0002 + ports: + - "18443:18443" + # bitcoin-cli: + # image: "ruimarinho/bitcoin-core:22" + # command: + # - /bin/sh + # - -c + # - | + # bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword createwallet Alice + # ALICE_ADDRESS=$$(bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword -rpcwallet=Alice getnewaddress) + # # coins need 100 confirmations to be spendable + # bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword generatetoaddress 101 $${ALICE_ADDRESS} + electrs: + image: "interlayhq/electrs:latest" + command: + - electrs + - -vvvv + - --network + - regtest + - --jsonrpc-import + - --cors + - "*" + - --cookie + - "rpcuser:rpcpassword" + - --daemon-rpc-addr + - "bitcoind:18443" + - --http-addr + - "[::0]:3002" + - --index-unspendables + ports: + - "3002:3002" + restart: always diff --git a/vault/src/execution.rs b/vault/src/execution.rs index a28292869..afbc8ce90 100644 --- a/vault/src/execution.rs +++ b/vault/src/execution.rs @@ -285,7 +285,7 @@ impl Request { .filter_map(|x| async { match btc_rpc.is_in_mempool(txid_copy).await { Ok(false) => { - // if not in mempool anymore, don't propagate the event (even if it is an error) + // if not in mempool anymore, don't propagate the event (even if it is an error) tracing::debug!("Txid not in mempool anymore..."); None } @@ -418,6 +418,48 @@ impl Request { } } +fn create_payment_worker( + shutdown_tx: ShutdownSender, + parachain_rpc: InterBtcParachain, + vault_id_manager: VaultIdManager, + request: Request, + txid: Txid, + num_confirmations: u32, + auto_rbf: bool, +) { + tracing::info!( + "{:?} request #{:?} has valid bitcoin payment - processing...", + request.request_type, + request.hash + ); + spawn_cancelable(shutdown_tx.subscribe(), async move { + let btc_rpc = match vault_id_manager.get_bitcoin_rpc(&request.vault_id).await { + Some(x) => x, + None => { + tracing::error!( + "Failed to fetch bitcoin rpc for vault {}", + request.vault_id.pretty_print() + ); + return; // nothing we can do - bail + } + }; + + match request + .wait_for_inclusion(¶chain_rpc, &btc_rpc, num_confirmations, txid, auto_rbf) + .await + { + Ok(tx_metadata) => { + if let Err(e) = request.execute(parachain_rpc.clone(), tx_metadata).await { + tracing::error!("Failed to execute request #{}: {}", request.hash, e); + } + } + Err(e) => { + tracing::error!("Error while waiting for inclusion for request #{}: {}", request.hash, e); + } + } + }); +} + /// Queries the parachain for open requests and executes them. It checks the /// bitcoin blockchain to see if a payment has already been made. #[allow(clippy::too_many_arguments)] @@ -465,82 +507,84 @@ pub async fn execute_open_requests( None => return Ok(()), // the iterator is empty so we have nothing to do }; - let rate_limiter = RateLimiter::direct(YIELD_RATE); + // NOTE: block iteration is expensive so only use a full node + // direct lookup is possible with the light client + if read_only_btc_rpc.is_full_node() { + let rate_limiter = RateLimiter::direct(YIELD_RATE); + // iterate through transactions in reverse order, starting from those in the mempool, and + // gracefully fail on encountering a pruned blockchain + let mut transaction_stream = bitcoin::reverse_stream_transactions(&read_only_btc_rpc, btc_start_height).await?; + while let Some(result) = transaction_stream.next().await { + if rate_limiter.check().is_ok() { + // give the outer `select` a chance to check the shutdown signal + tokio::task::yield_now().await; + } + + // When there is an error, we have to make a choice. Either we restart or + // continue processing. Both options have their risks: if we restart, it's + // possible that the vault will never manage to start up, causing us to + // fail to process requests. On the other hand, if we ignore transactions, + // we risk double paying. We choose to restart only for network errors, + // since these are not expected to be persistent. Other errors could be + // persistent, so we keep going. + let tx = match result { + Ok(x) => x, + Err(e) if e.is_transport_error() => { + return Err(e.into()); + } + Err(e) => { + tracing::warn!("Failed to process transaction: {}", e); + continue; + } + }; - // iterate through transactions in reverse order, starting from those in the mempool, and - // gracefully fail on encountering a pruned blockchain - let mut transaction_stream = bitcoin::reverse_stream_transactions(&read_only_btc_rpc, btc_start_height).await?; - while let Some(result) = transaction_stream.next().await { - if rate_limiter.check().is_ok() { - // give the outer `select` a chance to check the shutdown signal - tokio::task::yield_now().await; + // get the request this transaction corresponds to, if any + if let Some(request) = get_request_for_btc_tx(&tx, &open_requests) { + // remove request from the hashmap + open_requests.retain(|&key, _| key != request.hash); + + // start a new task to (potentially) await confirmation and to execute on the parachain + // make copies of the variables we move into the task + create_payment_worker( + shutdown_tx.clone(), + parachain_rpc.clone(), + vault_id_manager.clone(), + request, + tx.txid(), + num_confirmations, + auto_rbf, + ); + } } + } - // When there is an error, we have to make a choice. Either we restart or - // continue processing. Both options have their risks: if we restart, it's - // possible that the vault will never manage to start up, causing us to - // fail to process requests. On the other hand, if we ignore transactions, - // we risk double paying. We choose to restart only for network errors, - // since these are not expected to be persistent. Other errors could be - // persistent, so we keep going. - let tx = match result { - Ok(x) => x, - Err(e) if e.is_transport_error() => { - return Err(e.into()); + // All requests remaining in the hashmap did not have a bitcoin payment yet + // or were not found in the stream, check if we can fetch by id directly + // or just pay and execute all requests individually + for (id, request) in open_requests { + // try finding transaction directly (if supported) + match read_only_btc_rpc.get_tx_for_op_return(id).await { + Ok(Some(txid)) => { + create_payment_worker( + shutdown_tx.clone(), + parachain_rpc.clone(), + vault_id_manager.clone(), + request, + txid, + num_confirmations, + auto_rbf, + ); + // task will handling execution + continue; } - Err(e) => { - tracing::warn!("Failed to process transaction: {}", e); + Ok(None) => {} // make payment + Err(err) => { + tracing::error!("Failed to fetch tx for OP_RETURN {}", err); + // TODO: can we handle this error and still make the payment? continue; } - }; - - // get the request this transaction corresponds to, if any - if let Some(request) = get_request_for_btc_tx(&tx, &open_requests) { - // remove request from the hashmap - open_requests.retain(|&key, _| key != request.hash); - - tracing::info!( - "{:?} request #{:?} has valid bitcoin payment - processing...", - request.request_type, - request.hash - ); - - // start a new task to (potentially) await confirmation and to execute on the parachain - // make copies of the variables we move into the task - let parachain_rpc = parachain_rpc.clone(); - let btc_rpc = vault_id_manager.clone(); - spawn_cancelable(shutdown_tx.subscribe(), async move { - let btc_rpc = match btc_rpc.get_bitcoin_rpc(&request.vault_id).await { - Some(x) => x, - None => { - tracing::error!( - "Failed to fetch bitcoin rpc for vault {}", - request.vault_id.pretty_print() - ); - return; // nothing we can do - bail - } - }; - - match request - .wait_for_inclusion(¶chain_rpc, &btc_rpc, num_confirmations, tx.txid(), auto_rbf) - .await - { - Ok(tx_metadata) => { - if let Err(e) = request.execute(parachain_rpc.clone(), tx_metadata).await { - tracing::error!("Failed to execute request #{}: {}", request.hash, e); - } - } - Err(e) => { - tracing::error!("Error while waiting for inclusion for request #{}: {}", request.hash, e); - } - } - }); } - } - // All requests remaining in the hashmap did not have a bitcoin payment yet, so pay - // and execute all of these - for (_, request) in open_requests { // there are potentially a large number of open requests - pay and execute each // in a separate task to ensure that awaiting confirmations does not significantly // delay other requests @@ -734,6 +778,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; diff --git a/vault/src/issue.rs b/vault/src/issue.rs index a9207f6d9..73203c7b6 100644 --- a/vault/src/issue.rs +++ b/vault/src/issue.rs @@ -44,6 +44,8 @@ pub async fn process_issue_requests( num_confirmations: u32, random_delay: Arc>, ) -> Result<(), ServiceError> { + // NOTE: we should not stream transactions if using the light client + // since it is quite expensive to fetch all transactions per block let mut stream = bitcoin::stream_in_chain_transactions(bitcoin_core.clone(), btc_start_height, num_confirmations).await; diff --git a/vault/src/metrics.rs b/vault/src/metrics.rs index 0be4739e0..831293e05 100644 --- a/vault/src/metrics.rs +++ b/vault/src/metrics.rs @@ -790,6 +790,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; diff --git a/vault/src/replace.rs b/vault/src/replace.rs index a3f1b7c9d..214ac4e6e 100644 --- a/vault/src/replace.rs +++ b/vault/src/replace.rs @@ -233,6 +233,7 @@ mod tests { #[async_trait] trait BitcoinCoreApi { + fn is_full_node(&self) -> bool; fn network(&self) -> Network; async fn wait_for_block(&self, height: u32, num_confirmations: u32) -> Result; fn get_balance(&self, min_confirmations: Option) -> Result; diff --git a/vault/src/system.rs b/vault/src/system.rs index b4480ffe1..64cf2bffa 100644 --- a/vault/src/system.rs +++ b/vault/src/system.rs @@ -750,7 +750,7 @@ impl VaultService { ( "Issue Executor", maybe_run( - !self.config.no_issue_execution, + !self.config.no_issue_execution && self.btc_rpc_master_wallet.is_full_node(), issue::process_issue_requests( self.btc_rpc_master_wallet.clone(), self.btc_parachain.clone(), From 9ad4fa87d8f53bdef849a97dedf23625f0225bbb Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 22 Mar 2023 14:56:59 +0000 Subject: [PATCH 02/13] chore: add tests for light client Signed-off-by: Gregory Hill --- .env | 1 + Cargo.lock | 10 +- bitcoin/src/electrs/mod.rs | 74 +++++++++++---- bitcoin/src/iter.rs | 2 +- bitcoin/src/lib.rs | 9 +- bitcoin/src/light/mod.rs | 19 ++-- bitcoin/src/light/wallet.rs | 2 +- bitcoin/tests/electrs.rs | 184 ++++++++++++++++++++++++++++++++++++ docker-compose.yml | 20 ++-- vault/src/execution.rs | 2 +- 10 files changed, 274 insertions(+), 49 deletions(-) create mode 100644 bitcoin/tests/electrs.rs diff --git a/.env b/.env index 87495da4e..04a9a5f7f 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ export BITCOIN_RPC_URL=http://localhost:18443 export BITCOIN_RPC_USER=rpcuser export BITCOIN_RPC_PASS=rpcpassword +export ELECTRS_URL=http://localhost:3002 \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index c28defba4..67dced928 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15984,12 +15984,12 @@ dependencies = [ "pkg-config", ] -[[patch.unused]] -name = "orml-xcm" -version = "0.4.1-dev" -source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" - [[patch.unused]] name = "sp-serializer" version = "4.0.0-dev" source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" + +[[patch.unused]] +name = "orml-xcm" +version = "0.4.1-dev" +source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index 8c5b4fc6e..64588e602 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -12,6 +12,7 @@ use reqwest::{Client, Url}; use sha2::{Digest, Sha256}; use std::str::FromStr; +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L42 const ELECTRS_TRANSACTIONS_PER_PAGE: usize = 25; // https://github.com/Blockstream/esplora/blob/master/API.md @@ -86,7 +87,7 @@ impl ElectrsClient { )?) } - pub(crate) async fn get_address_tx_history_full(&self, address: &str) -> Result, Error> { + pub async fn get_address_tx_history_full(&self, address: &str) -> Result, Error> { let mut last_seen_txid = Default::default(); let mut ret = Vec::::new(); loop { @@ -104,7 +105,7 @@ impl ElectrsClient { Ok(ret) } - pub(crate) async fn get_blocks_tip_height(&self) -> Result { + pub async fn get_blocks_tip_height(&self) -> Result { Ok(self.get("/blocks/tip/height").await?.parse()?) } @@ -113,7 +114,7 @@ impl ElectrsClient { Ok(BlockHash::from_str(&response)?) } - pub(crate) async fn get_block_header(&self, hash: &BlockHash) -> Result { + pub async fn get_block_header(&self, hash: &BlockHash) -> Result { let raw_block_header = Vec::::from_hex(&self.get(&format!("/block/{hash}/header")).await?)?; Ok(deserialize(&raw_block_header)?) } @@ -140,7 +141,7 @@ impl ElectrsClient { Ok(Block { header, txdata }) } - pub(crate) async fn get_block_hash(&self, height: u32) -> Result { + pub async fn get_block_hash(&self, height: u32) -> Result { let response = self.get(&format!("/block-height/{height}")).await?; Ok(BlockHash::from_str(&response)?) } @@ -168,7 +169,7 @@ impl ElectrsClient { }) } - pub(crate) async fn get_utxos_for_address(&self, address: Address) -> Result, Error> { + pub async fn get_utxos_for_address(&self, address: &Address) -> Result, Error> { let utxos: Vec = self.get_and_decode(&format!("/address/{address}/utxo")).await?; utxos @@ -200,6 +201,8 @@ impl ElectrsClient { )?) } + // TODO: modify upstream to return a human-readable error + // or maybe add an endpoint for `testmempoolaccept` pub(crate) async fn send_transaction(&self, tx: Transaction) -> Result { let url = self.url.join("/tx")?; let txid = self @@ -214,7 +217,12 @@ impl ElectrsClient { Ok(Txid::from_str(&txid)?) } - pub(crate) async fn get_tx_for_op_return(&self, data: H256) -> Result, Error> { + pub(crate) async fn get_tx_for_op_return( + &self, + address: Address, + amount: u128, + data: H256, + ) -> Result, Error> { let script = ScriptBuilder::new() .push_opcode(opcodes::OP_RETURN) .push_slice(data.as_bytes()) @@ -226,24 +234,48 @@ impl ElectrsClient { hasher.result().as_slice().to_vec() }; - // TODO: page this using last_seen_txid - // NOTE: includes unconfirmed txs - let txs: Vec = self - .get_and_decode(&format!( - "/scripthash/{scripthash}/txs", - scripthash = script_hash.to_hex() - )) - .await?; + let mut last_seen_txid = Default::default(); + let mut txs = Vec::::new(); + loop { + // NOTE: includes unconfirmed txs + let mut transactions: Vec = self + .get_and_decode(&format!( + "/scripthash/{scripthash}/txs/chain/{last_seen_txid}", + scripthash = script_hash.to_hex() + )) + .await?; + let page_size = transactions.len(); + last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.clone()); + txs.append(&mut transactions); + if page_size < ELECTRS_TRANSACTIONS_PER_PAGE { + // no further pages + break; + } + } log::info!("Found {} transactions", txs.len()); - // TODO: check payment amount or throw error - // if there are multiple transactions - if let Some(tx) = txs.first().cloned() { - let txid = Txid::from_str(&tx.txid)?; - Ok(Some(txid)) - } else { - Ok(None) + let address = address.to_string(); + for tx in txs { + let largest = tx + .vout + .unwrap_or_default() + .iter() + .filter_map(|vout| { + if vout.scriptpubkey_address.contains(&address) { + Some(vout.value.unwrap_or_default() as u64) + } else { + None + } + }) + .max() + .unwrap_or_default(); + + if largest as u128 >= amount { + let txid = Txid::from_str(&tx.txid)?; + return Ok(Some(txid)); + } } + Ok(None) } } diff --git a/bitcoin/src/iter.rs b/bitcoin/src/iter.rs index d49d86023..311fa5c92 100644 --- a/bitcoin/src/iter.rs +++ b/bitcoin/src/iter.rs @@ -263,7 +263,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; - async fn get_tx_for_op_return(&self, data: H256) -> Result, Error>; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, Error>; } } diff --git a/bitcoin/src/lib.rs b/bitcoin/src/lib.rs index e0cd067d1..0ca430ee8 100644 --- a/bitcoin/src/lib.rs +++ b/bitcoin/src/lib.rs @@ -1,4 +1,5 @@ #![feature(int_roundings)] +#![feature(option_result_contains)] pub mod cli; pub mod light; @@ -205,11 +206,11 @@ pub trait BitcoinCoreApi { async fn fee_rate(&self, txid: Txid) -> Result; - async fn get_tx_for_op_return(&self, data: H256) -> Result, Error>; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, Error>; } -struct LockedTransaction { - transaction: Transaction, +pub struct LockedTransaction { + pub transaction: Transaction, recipient: String, _lock: Option>, } @@ -1077,7 +1078,7 @@ impl BitcoinCoreApi for BitcoinCore { Ok(SatPerVbyte(fee_rate.try_into()?)) } - async fn get_tx_for_op_return(&self, _data: H256) -> Result, Error> { + async fn get_tx_for_op_return(&self, _address: Address, _amount: u128, _data: H256) -> Result, Error> { // direct lookup not supported by bitcoin core Ok(None) } diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 340a3d246..1c021c806 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -25,14 +25,16 @@ pub struct BitcoinLight { impl BitcoinLight { pub fn new(electrs_url: Option, private_key: PrivateKey) -> Result { let network = private_key.network; - log::info!("Using network: {}", network); let electrs_client = ElectrsClient::new(electrs_url, network)?; + let wallet = wallet::Wallet::new(network, electrs_client.clone()); + // store the derivation key so it can be used for change + wallet.put_p2wpkh_key(private_key.inner.clone())?; Ok(Self { private_key, secp_ctx: secp256k1::Secp256k1::new(), - electrs: electrs_client.clone(), + electrs: electrs_client, transaction_creation_lock: Arc::new(Mutex::new(())), - wallet: wallet::Wallet::new(network, electrs_client), + wallet, }) } @@ -45,7 +47,7 @@ impl BitcoinLight { .ok_or(Error::NoChangeAddress) } - async fn create_transaction( + pub async fn create_transaction( &self, recipient: Address, sat: u64, @@ -314,7 +316,12 @@ impl BitcoinCoreApi for BitcoinLight { Ok(SatPerVbyte(fee_rate)) } - async fn get_tx_for_op_return(&self, data: H256) -> Result, BitcoinError> { - Ok(self.electrs.get_tx_for_op_return(data).await?) + async fn get_tx_for_op_return( + &self, + address: Address, + amount: u128, + data: H256, + ) -> Result, BitcoinError> { + Ok(self.electrs.get_tx_for_op_return(address, amount, data).await?) } } diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index dc7f4b65a..9f31f2270 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -292,7 +292,7 @@ impl Wallet { for address in addresses { log::info!("Found address: {}", address); // get utxos for address - let utxos = self.electrs.get_utxos_for_address(address).await?; + let utxos = self.electrs.get_utxos_for_address(&address).await?; // TODO: stream this, no need to fetch for utxo in utxos { log::info!("Found utxo: {}", utxo.outpoint.txid); diff --git a/bitcoin/tests/electrs.rs b/bitcoin/tests/electrs.rs new file mode 100644 index 000000000..7964439b9 --- /dev/null +++ b/bitcoin/tests/electrs.rs @@ -0,0 +1,184 @@ +// #![cfg(feature = "uses-bitcoind")] + +use bitcoin::{ + secp256k1::{constants::SECRET_KEY_SIZE, Secp256k1}, + Address, AddressType, Amount, Auth, BitcoinCoreApi, BitcoinLight, BlockHash, Client, ElectrsClient, Error, Hash, + Network, PrivateKey, PublicKey, RpcApi, SatPerVbyte, SecretKey, H256, +}; +use futures::{future::join, Future}; +use rand::{thread_rng, Rng}; +use std::{env::var, time::Duration}; +use tokio::time::{sleep, timeout}; + +const DEFAULT_NETWORK: Network = Network::Regtest; + +fn new_random_key_pair() -> (PrivateKey, PublicKey) { + // NOTE: private key wif cannot encode regtest + let raw_secret_key: [u8; SECRET_KEY_SIZE] = thread_rng().gen(); + let secret_key = SecretKey::from_slice(&raw_secret_key).unwrap(); + let private_key = PrivateKey::new(secret_key, DEFAULT_NETWORK); + let public_key = PublicKey::from_private_key(&Secp256k1::new(), &private_key); + (private_key, public_key) +} + +fn new_bitcoin_client() -> Client { + Client::new( + &var("BITCOIN_RPC_URL").expect("BITCOIN_RPC_URL not set"), + Auth::UserPass( + var("BITCOIN_RPC_USER").expect("BITCOIN_RPC_USER not set"), + var("BITCOIN_RPC_PASS").expect("BITCOIN_RPC_PASS not set"), + ), + ) + .unwrap() +} + +fn new_bitcoin_light() -> BitcoinLight { + BitcoinLight::new( + Some(var("ELECTRS_URL").expect("ELECTRS_URL not set")), + new_random_key_pair().0, + ) + .unwrap() +} + +fn new_electrs() -> ElectrsClient { + ElectrsClient::new(Some(var("ELECTRS_URL").expect("ELECTRS_URL not set")), DEFAULT_NETWORK).unwrap() +} + +async fn wait_for_success(f: F) -> T +where + F: Fn() -> R, + R: Future>, +{ + timeout(Duration::from_secs(20), async move { + loop { + match f().await { + Ok(x) => return x, + Err(_) => sleep(Duration::from_millis(10)).await, + } + } + }) + .await + .expect("Time limit elapsed") +} + +fn mine_blocks(block_num: u64, maybe_address: Option
) -> BlockHash { + let bitcoin_client = new_bitcoin_client(); + let address = + maybe_address.unwrap_or_else(|| bitcoin_client.get_new_address(None, Some(AddressType::Bech32)).unwrap()); + bitcoin_client + .generate_to_address(block_num, &address) + .unwrap() + .last() + .unwrap() + .clone() +} + +#[tokio::test] +async fn should_create_transactions() -> Result<(), Error> { + let bitcoin_client = new_bitcoin_client(); + let bitcoin_light = new_bitcoin_light(); + + // need at least 100 confirmations otherwise we get + // this error: bad-txns-premature-spend-of-coinbase + let public_key = new_random_key_pair().1; + let address = Address::p2wpkh(&public_key, DEFAULT_NETWORK).unwrap(); + mine_blocks(101, Some(address)); + + // fund the master key + let master_public_key = bitcoin_light.get_new_public_key().await?; + let master_address = Address::p2wpkh(&master_public_key, DEFAULT_NETWORK).unwrap(); + // electrs may still include unconfirmed coinbase txs so to + // avoid this we mine and then send it to the master address + bitcoin_client.send_to_address( + &master_address, + Amount::from_sat(100000), + None, + None, + None, + None, + None, + None, + )?; + mine_blocks(1, None); + + // wait for electrs to pickup utxo + wait_for_success(|| async { + new_electrs() + .get_utxos_for_address(&master_address) + .await + .map_err(|_| ())? + .len() + .gt(&0) + .then_some(()) + .ok_or(()) + }) + .await; + + let address1 = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); + let address2 = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); + let (res1, res2) = join( + bitcoin_light.create_and_send_transaction( + address1.clone(), + 1000, + SatPerVbyte(1), + Some(H256::from_slice(&[1; 32])), + ), + bitcoin_light.create_and_send_transaction( + address2.clone(), + 1000, + SatPerVbyte(1), + Some(H256::from_slice(&[2; 32])), + ), + ) + .await; + + let txid1 = res1?; + let txid2 = res2?; + + let mempool_txs: Vec<_> = bitcoin_light.get_mempool_transactions().await?.collect(); + assert_eq!(mempool_txs.len(), 2); + assert!(bitcoin_light.is_in_mempool(txid1).await?, "Txid1 not in mempool"); + assert!(bitcoin_light.is_in_mempool(txid2).await?, "Txid2 not in mempool"); + + // mine a block to include our transactions + let block_hash = mine_blocks(1, None); + // wait for electrs to pickup block + wait_for_success(|| async { bitcoin_light.get_block(&block_hash).await }).await; + + assert!( + bitcoin_light.get_proof(txid1, &BlockHash::all_zeros()).await.is_ok(), + "Txid1 not confirmed" + ); + assert!( + bitcoin_light.get_proof(txid2, &BlockHash::all_zeros()).await.is_ok(), + "Txid2 not confirmed" + ); + + assert!(bitcoin_light + .get_tx_for_op_return(address1, 1000, H256::from_slice(&[1; 32])) + .await? + .is_some()); + + assert!(bitcoin_light + .get_tx_for_op_return(address2, 1000, H256::from_slice(&[2; 32])) + .await? + .is_some()); + + Ok(()) +} + +#[tokio::test] +async fn should_page_address_history() -> Result<(), Error> { + let public_key = new_random_key_pair().1; + let address = Address::p2wpkh(&public_key, DEFAULT_NETWORK).unwrap(); + let block_hash = mine_blocks(100, Some(address.clone())); + wait_for_success(|| async { new_electrs().get_block_header(&block_hash).await }).await; + + let txs = new_electrs().get_address_tx_history_full(&address.to_string()).await?; + // ideally we would check that 100 coinbase txs are + // returned but this test may run concurrently so we + // may not have cached all txs + assert!(txs.len().gt(&25)); + + Ok(()) +} diff --git a/docker-compose.yml b/docker-compose.yml index d50fc3bba..d4b66012a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,16 +12,16 @@ services: - -fallbackfee=0.0002 ports: - "18443:18443" - # bitcoin-cli: - # image: "ruimarinho/bitcoin-core:22" - # command: - # - /bin/sh - # - -c - # - | - # bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword createwallet Alice - # ALICE_ADDRESS=$$(bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword -rpcwallet=Alice getnewaddress) - # # coins need 100 confirmations to be spendable - # bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword generatetoaddress 101 $${ALICE_ADDRESS} + bitcoin-cli: + image: "ruimarinho/bitcoin-core:22" + command: + - /bin/sh + - -c + - | + bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword createwallet Alice + ALICE_ADDRESS=$$(bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword -rpcwallet=Alice getnewaddress) + # coins need 100 confirmations to be spendable + bitcoin-cli -regtest -rpcconnect=bitcoind -rpcwait -rpcuser=rpcuser -rpcpassword=rpcpassword generatetoaddress 101 $${ALICE_ADDRESS} electrs: image: "interlayhq/electrs:latest" command: diff --git a/vault/src/execution.rs b/vault/src/execution.rs index afbc8ce90..fb37781e2 100644 --- a/vault/src/execution.rs +++ b/vault/src/execution.rs @@ -563,7 +563,7 @@ pub async fn execute_open_requests( // or just pay and execute all requests individually for (id, request) in open_requests { // try finding transaction directly (if supported) - match read_only_btc_rpc.get_tx_for_op_return(id).await { + match read_only_btc_rpc.get_tx_for_op_return(request.btc_address, request.amount, id).await { Ok(Some(txid)) => { create_payment_worker( shutdown_tx.clone(), From 326909f2113cd22730a4c24b82268966e700f45c Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Tue, 28 Mar 2023 13:14:04 +0000 Subject: [PATCH 03/13] feat: add fee bumping (rbf) to light client Signed-off-by: Gregory Hill --- Cargo.lock | 10 +- bitcoin/src/electrs/mod.rs | 17 +++- bitcoin/src/light/mod.rs | 55 ++++++++-- bitcoin/src/light/wallet.rs | 194 ++++++++++++++++++++++++------------ bitcoin/tests/electrs.rs | 44 ++++++-- 5 files changed, 233 insertions(+), 87 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67dced928..c28defba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15984,12 +15984,12 @@ dependencies = [ "pkg-config", ] -[[patch.unused]] -name = "sp-serializer" -version = "4.0.0-dev" -source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" - [[patch.unused]] name = "orml-xcm" version = "0.4.1-dev" source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" + +[[patch.unused]] +name = "sp-serializer" +version = "4.0.0-dev" +source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index 64588e602..fa37169d0 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -171,7 +171,7 @@ impl ElectrsClient { pub async fn get_utxos_for_address(&self, address: &Address) -> Result, Error> { let utxos: Vec = self.get_and_decode(&format!("/address/{address}/utxo")).await?; - + // NOTE: includes unconfirmed mempool txs utxos .into_iter() .map(|utxo| { @@ -201,6 +201,21 @@ impl ElectrsClient { )?) } + pub(crate) async fn get_prev_value(&self, outpoint: &OutPoint) -> Result { + let tx: ElectrsTransaction = self + .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) + .await?; + Ok(tx + .vout + .ok_or(Error::NoPrevOut)? + .get(outpoint.vout as usize) + .ok_or(Error::NoPrevOut)? + .clone() + .value + .map(|v| v as u64) + .ok_or(Error::NoPrevOut)?) + } + // TODO: modify upstream to return a human-readable error // or maybe add an endpoint for `testmempoolaccept` pub(crate) async fn send_transaction(&self, tx: Transaction) -> Result { diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 1c021c806..83b3e67ec 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -47,27 +47,37 @@ impl BitcoinLight { .ok_or(Error::NoChangeAddress) } - pub async fn create_transaction( + pub async fn fund_and_sign_transaction( &self, recipient: Address, - sat: u64, + unsigned_tx: Transaction, + change_address: Address, fee_rate: SatPerVbyte, - request_id: Option, + prev_txid: Option, ) -> Result { let lock = self.transaction_creation_lock.clone().lock_owned().await; - let unsigned_tx = self.wallet.create_transaction(recipient.clone(), sat, request_id); - let change_address = self.get_change_address()?; - let mut psbt = self .wallet - .fund_transaction(unsigned_tx, change_address, fee_rate.0.saturating_mul(1000)) + .fund_transaction(unsigned_tx, change_address, fee_rate.0.saturating_mul(1000), prev_txid) .await?; self.wallet.sign_transaction(&mut psbt)?; let signed_tx = psbt.extract_tx(); - Ok(LockedTransaction::new(signed_tx, recipient.to_string(), Some(lock))) } + pub async fn create_transaction( + &self, + recipient: Address, + sat: u64, + fee_rate: SatPerVbyte, + request_id: Option, + ) -> Result { + let unsigned_tx = self.wallet.create_transaction(recipient.clone(), sat, request_id); + let change_address = self.get_change_address()?; + self.fund_and_sign_transaction(recipient, unsigned_tx, change_address, fee_rate, None) + .await + } + // TODO: hold tx lock until inclusion otherwise electrs may report stale utxos // need to check when electrs knows that a utxo has been spent async fn send_transaction(&self, transaction: LockedTransaction) -> Result { @@ -239,8 +249,33 @@ impl BitcoinCoreApi for BitcoinLight { }) } - async fn bump_fee(&self, _txid: &Txid, _address: Address, _fee_rate: SatPerVbyte) -> Result { - todo!() + async fn bump_fee(&self, txid: &Txid, address: Address, fee_rate: SatPerVbyte) -> Result { + let mut existing_transaction = self.get_transaction(txid, None).await?; + let return_to_self = existing_transaction + .extract_return_to_self_address(&address.payload)? + .map(|(idx, payload)| { + existing_transaction.output.remove(idx); + Address { + payload, + network: self.network(), + } + }); + + existing_transaction + .input + .iter_mut() + .for_each(|txin| txin.witness.clear()); + let tx = self + .fund_and_sign_transaction( + address, + existing_transaction, + return_to_self.unwrap(), + fee_rate, + Some(txid.clone()), + ) + .await?; + let txid = self.send_transaction(tx).await?; + Ok(txid) } async fn create_and_send_transaction( diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index 9f31f2270..465ad4e4b 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -1,11 +1,6 @@ -use bitcoincore_rpc::bitcoin::{ - blockdata::{constants::WITNESS_SCALE_FACTOR, transaction::NonStandardSighashType}, - util::sighash::SighashCache, - EcdsaSig, PackedLockTime, PublicKey, Witness, -}; - use super::{electrs::ElectrsClient, error::Error}; use crate::{ + electrs::Utxo, hashes::Hash, json::bitcoin::EcdsaSighashType, opcodes, psbt, @@ -14,8 +9,14 @@ use crate::{ Address, Builder as ScriptBuilder, Network, OutPoint, PrivateKey, Script, Transaction, TxIn, TxOut, Txid, VarInt, H256, }; +use bitcoincore_rpc::bitcoin::{ + blockdata::{constants::WITNESS_SCALE_FACTOR, transaction::NonStandardSighashType}, + util::sighash::SighashCache, + EcdsaSig, PackedLockTime, PublicKey, Sequence, Witness, +}; +use futures::{stream, Stream, StreamExt}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, VecDeque}, sync::{Arc, RwLock}, }; @@ -215,6 +216,63 @@ fn get_dust_threshold(tx_out: &TxOut, dust_relay_fee_in: &FeeRate) -> u64 { pub type KeyStore = Arc>>; +pub fn available_coins( + electrs: ElectrsClient, + tx_inputs: Vec, + addresses: Vec
, +) -> impl Stream> + Unpin { + struct StreamState { + electrs: E, + utxos: VecDeque, + tx_inputs: VecDeque, + addresses: VecDeque
, + } + + let state = StreamState { + electrs, + utxos: VecDeque::new(), + tx_inputs: VecDeque::from(tx_inputs), + addresses: VecDeque::from(addresses), + }; + + Box::pin( + stream::unfold(state, move |mut state| async move { + match state.utxos.pop_front() { + Some(utxo) => Some((Ok(utxo), state)), + None => { + if let Some(txin) = state.tx_inputs.pop_front() { + match state.electrs.get_prev_value(&txin.previous_output).await { + Ok(value) => Some(( + Ok(Utxo { + outpoint: txin.previous_output, + value, + }), + state, + )), + Err(e) => Some((Err(Error::ElectrsError(e)), state)), + } + } else if let Some(address) = state.addresses.pop_front() { + match state.electrs.get_utxos_for_address(&address).await { + Ok(utxos) => { + state.utxos = VecDeque::from(utxos); + if let Some(utxo) = state.utxos.pop_front() { + Some((Ok(utxo), state)) + } else { + None + } + } + Err(e) => Some((Err(Error::ElectrsError(e)), state)), + } + } else { + None + } + } + } + }) + .fuse(), + ) +} + #[derive(Clone)] pub struct Wallet { secp: Secp256k1, @@ -249,6 +307,7 @@ impl Wallet { tx: Transaction, change_address: Address, n_satoshis_per_k: u64, + prev_txid: Option, ) -> Result { let recipients_sum = tx.output.iter().map(|tx_out| tx_out.value).sum::(); @@ -283,73 +342,78 @@ impl Wallet { // https://github.com/bitcoin/bitcoin/blob/01e1627e25bc5477c40f51da03c3c31b609a85c9/src/wallet/spend.cpp#L896 let selection_target = recipients_sum + not_input_fees; let mut value_to_select = selection_target; - - let mut psbt = PartiallySignedTransaction::from_unsigned_tx(tx)?; let mut select_coins = SelectCoins::new(selection_target); + // not empty if we are fee bumping, include these inputs first + let prev_inputs = tx.input.clone(); + let mut tx_noinputs = tx.clone(); + tx_noinputs.input.clear(); + let mut psbt = PartiallySignedTransaction::from_unsigned_tx(tx_noinputs)?; + // get available coins let addresses = self.key_store.read()?.keys().cloned().collect::>(); - for address in addresses { - log::info!("Found address: {}", address); - // get utxos for address - let utxos = self.electrs.get_utxos_for_address(&address).await?; - // TODO: stream this, no need to fetch - for utxo in utxos { - log::info!("Found utxo: {}", utxo.outpoint.txid); - - let script_pubkey = self.electrs.get_script_pubkey(utxo.outpoint).await?; - let public_key = self.get_pub_key(&script_pubkey).expect("wallet has key"); - let input_bytes = calculate_maximum_signed_input_size(utxo.outpoint, public_key); - let coin_output = CoinOutput { + let mut utxo_stream = available_coins(self.electrs.clone(), prev_inputs, addresses); + while let Some(Ok(utxo)) = utxo_stream.next().await { + log::info!("Found utxo: {}", utxo.outpoint.txid); + + if prev_txid.contains(&utxo.outpoint.txid) { + // skip if trying to spend previous tx (RBF) + continue; + } + + let script_pubkey = self.electrs.get_script_pubkey(utxo.outpoint).await?; + let public_key = self.get_pub_key(&script_pubkey).expect("wallet has key"); + let input_bytes = calculate_maximum_signed_input_size(utxo.outpoint, public_key); + let coin_output = CoinOutput { + value: utxo.value, + fee: m_effective_feerate.get_fee(input_bytes), + }; + + let effective_value = coin_output.get_effective_value(); + select_coins.add(coin_output); + value_to_select = value_to_select.saturating_sub(effective_value); + + psbt.unsigned_tx.input.push(TxIn { + previous_output: utxo.outpoint, + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, + ..Default::default() + }); + + psbt.inputs.push(psbt::Input { + witness_utxo: Some(TxOut { value: utxo.value, - fee: m_effective_feerate.get_fee(input_bytes), - }; - - let effective_value = coin_output.get_effective_value(); - select_coins.add(coin_output); - value_to_select = value_to_select.saturating_sub(effective_value); - - psbt.unsigned_tx.input.push(TxIn { - previous_output: utxo.outpoint, - ..Default::default() - }); - - psbt.inputs.push(psbt::Input { - witness_utxo: Some(TxOut { - value: utxo.value, - script_pubkey, - }), - ..Default::default() - }); - - if value_to_select == 0 { - // add change output before computing maximum size - let change_amount = select_coins.get_change(min_viable_change, change_fee); - let mut n_change_pos_in_out = None; - if change_amount > 0 { - n_change_pos_in_out = Some(psbt.unsigned_tx.output.len()); - // add change output - psbt.unsigned_tx.output.push(TxOut { - value: change_amount, - script_pubkey: change_address.script_pubkey(), - }); - } + script_pubkey, + }), + ..Default::default() + }); + + if value_to_select == 0 { + // add change output before computing maximum size + let change_amount = select_coins.get_change(min_viable_change, change_fee); + let mut n_change_pos_in_out = None; + if change_amount > 0 { + n_change_pos_in_out = Some(psbt.unsigned_tx.output.len()); + // add change output + psbt.unsigned_tx.output.push(TxOut { + value: change_amount, + script_pubkey: change_address.script_pubkey(), + }); + } - // https://github.com/bitcoin/bitcoin/blob/01e1627e25bc5477c40f51da03c3c31b609a85c9/src/wallet/spend.cpp#L945 - let n_bytes = calculate_maximum_signed_tx_size(&psbt, self); - let fee_needed = m_effective_feerate.get_fee(n_bytes); - let n_fee_ret = select_coins.get_selected_value() - recipients_sum - change_amount; + // https://github.com/bitcoin/bitcoin/blob/01e1627e25bc5477c40f51da03c3c31b609a85c9/src/wallet/spend.cpp#L945 + let n_bytes = calculate_maximum_signed_tx_size(&psbt, self); + let fee_needed = m_effective_feerate.get_fee(n_bytes); + let n_fee_ret = select_coins.get_selected_value() - recipients_sum - change_amount; - if let Some(change_pos) = n_change_pos_in_out { - if fee_needed < n_fee_ret { - log::info!("Fee needed is less than expected"); - let mut change_output = &mut psbt.unsigned_tx.output[change_pos]; - change_output.value += n_fee_ret - fee_needed; - } + if let Some(change_pos) = n_change_pos_in_out { + if fee_needed < n_fee_ret { + log::info!("Fee needed is less than expected"); + let mut change_output = &mut psbt.unsigned_tx.output[change_pos]; + change_output.value += n_fee_ret - fee_needed; } - - return Ok(psbt); } + + return Ok(psbt); } } diff --git a/bitcoin/tests/electrs.rs b/bitcoin/tests/electrs.rs index 7964439b9..05968b048 100644 --- a/bitcoin/tests/electrs.rs +++ b/bitcoin/tests/electrs.rs @@ -73,11 +73,7 @@ fn mine_blocks(block_num: u64, maybe_address: Option
) -> BlockHash { .clone() } -#[tokio::test] -async fn should_create_transactions() -> Result<(), Error> { - let bitcoin_client = new_bitcoin_client(); - let bitcoin_light = new_bitcoin_light(); - +async fn fund_wallet(bitcoin_light: &BitcoinLight) -> Result<(), Error> { // need at least 100 confirmations otherwise we get // this error: bad-txns-premature-spend-of-coinbase let public_key = new_random_key_pair().1; @@ -89,7 +85,7 @@ async fn should_create_transactions() -> Result<(), Error> { let master_address = Address::p2wpkh(&master_public_key, DEFAULT_NETWORK).unwrap(); // electrs may still include unconfirmed coinbase txs so to // avoid this we mine and then send it to the master address - bitcoin_client.send_to_address( + new_bitcoin_client().send_to_address( &master_address, Amount::from_sat(100000), None, @@ -113,6 +109,13 @@ async fn should_create_transactions() -> Result<(), Error> { .ok_or(()) }) .await; + Ok(()) +} + +#[tokio::test] +async fn should_create_transactions() -> Result<(), Error> { + let bitcoin_light = new_bitcoin_light(); + fund_wallet(&bitcoin_light).await?; let address1 = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); let address2 = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); @@ -167,6 +170,35 @@ async fn should_create_transactions() -> Result<(), Error> { Ok(()) } +#[tokio::test] +async fn should_bump_fee() -> Result<(), Error> { + let bitcoin_light = new_bitcoin_light(); + fund_wallet(&bitcoin_light).await?; + + let address = Address::p2wpkh(&new_random_key_pair().1, DEFAULT_NETWORK).unwrap(); + let txid1 = bitcoin_light + .create_and_send_transaction(address.clone(), 1000, SatPerVbyte(1), Some(H256::from_slice(&[1; 32]))) + .await?; + assert_eq!(SatPerVbyte(1), bitcoin_light.fee_rate(txid1).await?); + + let txid2 = bitcoin_light.bump_fee(&txid1, address, SatPerVbyte(2)).await?; + assert_eq!(SatPerVbyte(2), bitcoin_light.fee_rate(txid2).await?); + + let block_hash = mine_blocks(1, None); + wait_for_success(|| async { new_electrs().get_block_header(&block_hash).await }).await; + + assert!( + bitcoin_light.get_proof(txid1, &BlockHash::all_zeros()).await.is_err(), + "Txid1 should not exist" + ); + assert!( + bitcoin_light.get_proof(txid2, &BlockHash::all_zeros()).await.is_ok(), + "Txid2 should be confirmed" + ); + + Ok(()) +} + #[tokio::test] async fn should_page_address_history() -> Result<(), Error> { let public_key = new_random_key_pair().1; From 327074a832a7f2c9e01ba6ae10728895d4e8ec9f Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Tue, 28 Mar 2023 14:48:48 +0000 Subject: [PATCH 04/13] chore: use docker-compose for integration tests Signed-off-by: Gregory Hill --- .github/workflows/cargo-test.yml | 7 ++----- bitcoin/tests/electrs.rs | 2 +- vault/src/execution.rs | 12 +++++++++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 2e3056627..d51233e60 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -17,11 +17,6 @@ jobs: test: name: Test Suite runs-on: [self-hosted, linux] - services: - bitcoind: - image: docker.io/interlayhq/bitcoin-core:22.0 - ports: - - 18443:18443 strategy: matrix: @@ -62,7 +57,9 @@ jobs: BITCOIN_RPC_URL: http://127.0.0.1:18443 BITCOIN_RPC_USER: user BITCOIN_RPC_PASS: pass + ELECTRS_URL: http://localhost:3002 run: | + docker-compose up --detach cargo test --release --workspace --features ${{ matrix.metadata }} --features uses-bitcoind - name: build run: | diff --git a/bitcoin/tests/electrs.rs b/bitcoin/tests/electrs.rs index 05968b048..fc7804592 100644 --- a/bitcoin/tests/electrs.rs +++ b/bitcoin/tests/electrs.rs @@ -1,4 +1,4 @@ -// #![cfg(feature = "uses-bitcoind")] +#![cfg(feature = "uses-bitcoind")] use bitcoin::{ secp256k1::{constants::SECRET_KEY_SIZE, Secp256k1}, diff --git a/vault/src/execution.rs b/vault/src/execution.rs index fb37781e2..169f022b0 100644 --- a/vault/src/execution.rs +++ b/vault/src/execution.rs @@ -563,7 +563,17 @@ pub async fn execute_open_requests( // or just pay and execute all requests individually for (id, request) in open_requests { // try finding transaction directly (if supported) - match read_only_btc_rpc.get_tx_for_op_return(request.btc_address, request.amount, id).await { + match read_only_btc_rpc + .get_tx_for_op_return( + request + .btc_address + .to_address(read_only_btc_rpc.network()) + .map_err(BitcoinError::ConversionError)?, + request.amount, + id, + ) + .await + { Ok(Some(txid)) => { create_payment_worker( shutdown_tx.clone(), From 54ba6b92e0aef75823049024c9a53306a0150a3e Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 29 Mar 2023 11:44:07 +0000 Subject: [PATCH 05/13] chore: run electrs tests sequentially Signed-off-by: Gregory Hill --- Cargo.lock | 11 ++++++----- bitcoin/Cargo.toml | 1 + bitcoin/src/electrs/error.rs | 8 +++++++- bitcoin/src/light/mod.rs | 6 ++---- bitcoin/src/light/wallet.rs | 3 ++- bitcoin/tests/electrs.rs | 9 +++++---- runtime/src/integration/bitcoin_simulator.rs | 4 ++++ 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c28defba4..16df59fbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -613,6 +613,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serial_test", "sha2 0.8.2", "sp-core", "thiserror", @@ -15984,12 +15985,12 @@ dependencies = [ "pkg-config", ] -[[patch.unused]] -name = "orml-xcm" -version = "0.4.1-dev" -source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" - [[patch.unused]] name = "sp-serializer" version = "4.0.0-dev" source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" + +[[patch.unused]] +name = "orml-xcm" +version = "0.4.1-dev" +source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" diff --git a/bitcoin/Cargo.toml b/bitcoin/Cargo.toml index 4a9cf95d9..a8f28c9e2 100644 --- a/bitcoin/Cargo.toml +++ b/bitcoin/Cargo.toml @@ -42,3 +42,4 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot- mockall = "0.8.1" regex = "1.4.3" rand = "0.7" +serial_test = "*" \ No newline at end of file diff --git a/bitcoin/src/electrs/error.rs b/bitcoin/src/electrs/error.rs index 0757d18e2..e208f55b9 100644 --- a/bitcoin/src/electrs/error.rs +++ b/bitcoin/src/electrs/error.rs @@ -2,7 +2,7 @@ use bitcoincore_rpc::bitcoin::{ consensus::encode::Error as BitcoinEncodeError, hashes::hex::Error as HexError, util::address::Error as BitcoinAddressError, }; -use reqwest::Error as ReqwestError; +use reqwest::{Error as ReqwestError, StatusCode}; use serde_json::Error as SerdeJsonError; use std::num::{ParseIntError, TryFromIntError}; use thiserror::Error; @@ -36,3 +36,9 @@ pub enum Error { #[error("ParseIntError: {0}")] ParseIntError(#[from] ParseIntError), } + +impl Error { + pub fn is_not_found(&self) -> bool { + matches!(self, Error::ReqwestError(err) if err.status().contains(&StatusCode::NOT_FOUND)) + } +} diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 83b3e67ec..131655b0c 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -151,10 +151,8 @@ impl BitcoinCoreApi for BitcoinLight { async fn get_block_hash(&self, height: u32) -> Result { match self.electrs.get_block_hash(height).await { Ok(block_hash) => Ok(block_hash), - Err(_err) => { - // TODO: handle error - Err(BitcoinError::InvalidBitcoinHeight) - } + Err(err) if err.is_not_found() => Err(BitcoinError::InvalidBitcoinHeight), + Err(err) => Err(BitcoinError::ElectrsError(err)), } } diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index 465ad4e4b..2c71748bd 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -466,7 +466,8 @@ impl Wallet { .ecdsa_hash_ty() .map_err(|NonStandardSighashType(ty)| Error::PsbtError(psbt::Error::NonStandardSighashType(ty)))?; - // TODO: support signing p2sh, p2pkh, p2wsh + // NOTE: we don't support signing p2sh, p2pkh, p2wsh inputs + // since the Vault is assumed to only receive p2wpkh payments let script_code = if prev_out.script_pubkey.is_v0_p2wpkh() { Ok(p2wpkh_script_code(&prev_out.script_pubkey)) } else { diff --git a/bitcoin/tests/electrs.rs b/bitcoin/tests/electrs.rs index fc7804592..b7f31463b 100644 --- a/bitcoin/tests/electrs.rs +++ b/bitcoin/tests/electrs.rs @@ -7,6 +7,7 @@ use bitcoin::{ }; use futures::{future::join, Future}; use rand::{thread_rng, Rng}; +use serial_test::serial; use std::{env::var, time::Duration}; use tokio::time::{sleep, timeout}; @@ -113,6 +114,7 @@ async fn fund_wallet(bitcoin_light: &BitcoinLight) -> Result<(), Error> { } #[tokio::test] +#[serial] async fn should_create_transactions() -> Result<(), Error> { let bitcoin_light = new_bitcoin_light(); fund_wallet(&bitcoin_light).await?; @@ -171,6 +173,7 @@ async fn should_create_transactions() -> Result<(), Error> { } #[tokio::test] +#[serial] async fn should_bump_fee() -> Result<(), Error> { let bitcoin_light = new_bitcoin_light(); fund_wallet(&bitcoin_light).await?; @@ -200,6 +203,7 @@ async fn should_bump_fee() -> Result<(), Error> { } #[tokio::test] +#[serial] async fn should_page_address_history() -> Result<(), Error> { let public_key = new_random_key_pair().1; let address = Address::p2wpkh(&public_key, DEFAULT_NETWORK).unwrap(); @@ -207,10 +211,7 @@ async fn should_page_address_history() -> Result<(), Error> { wait_for_success(|| async { new_electrs().get_block_header(&block_hash).await }).await; let txs = new_electrs().get_address_tx_history_full(&address.to_string()).await?; - // ideally we would check that 100 coinbase txs are - // returned but this test may run concurrently so we - // may not have cached all txs - assert!(txs.len().gt(&25)); + assert!(txs.len().eq(&100)); Ok(()) } diff --git a/runtime/src/integration/bitcoin_simulator.rs b/runtime/src/integration/bitcoin_simulator.rs index 3cce14239..705161880 100644 --- a/runtime/src/integration/bitcoin_simulator.rs +++ b/runtime/src/integration/bitcoin_simulator.rs @@ -508,4 +508,8 @@ impl BitcoinCoreApi for MockBitcoinCore { async fn fee_rate(&self, txid: Txid) -> Result { unimplemented!() } + + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, Error> { + unimplemented!() + } } From 4a5e82d40c16d0fa10ad7ae1140a7ecd0cddaf7c Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 29 Mar 2023 12:45:48 +0000 Subject: [PATCH 06/13] refactor: duplicate electrs types and remove dep Signed-off-by: Gregory Hill --- Cargo.lock | 15 ----- bitcoin/Cargo.toml | 1 - bitcoin/src/electrs/mod.rs | 119 +++++++++++++++++------------------ bitcoin/src/electrs/types.rs | 67 ++++++++++++++++++++ bitcoin/src/lib.rs | 2 - 5 files changed, 123 insertions(+), 81 deletions(-) create mode 100644 bitcoin/src/electrs/types.rs diff --git a/Cargo.lock b/Cargo.lock index 16df59fbe..05db2d8d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,7 +599,6 @@ dependencies = [ "bitcoincore-rpc", "cfg-if 1.0.0", "clap", - "esplora-btc-api", "futures 0.3.26", "hex", "hyper 0.10.16", @@ -2850,19 +2849,6 @@ dependencies = [ "sp-api", ] -[[package]] -name = "esplora-btc-api" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0309de257f421df7081bfb9d5c3167adbd2c23c3589e0b6c5a2cdbdb826be65" -dependencies = [ - "reqwest", - "serde", - "serde_derive", - "serde_json", - "url 2.3.1", -] - [[package]] name = "event-listener" version = "2.5.3" @@ -10219,7 +10205,6 @@ dependencies = [ "js-sys", "log 0.4.17", "mime 0.3.16", - "mime_guess", "native-tls", "once_cell", "percent-encoding 2.2.0", diff --git a/bitcoin/Cargo.toml b/bitcoin/Cargo.toml index a8f28c9e2..3e7a90b0f 100644 --- a/bitcoin/Cargo.toml +++ b/bitcoin/Cargo.toml @@ -25,7 +25,6 @@ num-derive = "0.3" futures = "0.3.5" log = "0.4.0" hyper = "0.10" -esplora-btc-api = "1.0.3" sha2 = "0.8.2" cfg-if = "1.0" diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index fa37169d0..1f873e619 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -1,12 +1,13 @@ mod error; +mod types; pub use error::Error; +pub use types::*; use crate::{ deserialize, opcodes, serialize, Address, Block, BlockHash, BlockHeader, Builder as ScriptBuilder, FromHex, Network, OutPoint, Script, SignedAmount, ToHex, Transaction, Txid, H256, }; -use esplora_btc_api::models::{Transaction as ElectrsTransaction, Utxo as ElectrsUtxo}; use futures::future::{join_all, try_join}; use reqwest::{Client, Url}; use sha2::{Digest, Sha256}; @@ -69,33 +70,31 @@ impl ElectrsClient { Ok(serde_json::from_str(&body)?) } - pub(crate) async fn get_tx_hex(&self, txid: &str) -> Result { + pub(crate) async fn get_tx_hex(&self, txid: &Txid) -> Result { self.get(&format!("/tx/{txid}/hex")).await } - pub(crate) async fn get_tx_merkle_block_proof(&self, txid: &str) -> Result { + pub(crate) async fn get_tx_merkle_block_proof(&self, txid: &Txid) -> Result { self.get(&format!("/tx/{txid}/merkleblock-proof")).await } pub(crate) async fn get_raw_tx(&self, txid: &Txid) -> Result, Error> { - Ok(Vec::::from_hex(&self.get_tx_hex(&txid.to_string()).await?)?) + Ok(Vec::::from_hex(&self.get_tx_hex(txid).await?)?) } pub(crate) async fn get_raw_tx_merkle_proof(&self, txid: &Txid) -> Result, Error> { - Ok(Vec::::from_hex( - &self.get_tx_merkle_block_proof(&txid.to_string()).await?, - )?) + Ok(Vec::::from_hex(&self.get_tx_merkle_block_proof(txid).await?)?) } - pub async fn get_address_tx_history_full(&self, address: &str) -> Result, Error> { + pub async fn get_address_tx_history_full(&self, address: &str) -> Result, Error> { let mut last_seen_txid = Default::default(); - let mut ret = Vec::::new(); + let mut ret = Vec::::new(); loop { - let mut transactions: Vec = self + let mut transactions: Vec = self .get_and_decode(&format!("/address/{address}/txs/chain/{last_seen_txid}")) .await?; let page_size = transactions.len(); - last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.clone()); + last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.to_string()); ret.append(&mut transactions); if page_size < ELECTRS_TRANSACTIONS_PER_PAGE { // no further pages @@ -155,7 +154,7 @@ impl ElectrsClient { } pub(crate) async fn get_tx_info(&self, txid: &Txid) -> Result { - let tx: ElectrsTransaction = self.get_and_decode(&format!("/tx/{txid}")).await?; + let tx: TransactionValue = self.get_and_decode(&format!("/tx/{txid}")).await?; let tip = self.get_blocks_tip_height().await?; let (height, hash) = match tx.status.map(|status| (status.block_height, status.block_hash)) { Some((Some(height), Some(hash))) => (height as u32, hash), @@ -164,56 +163,50 @@ impl ElectrsClient { Ok(TxInfo { confirmations: tip.saturating_sub(height), height, - hash: BlockHash::from_str(&hash)?, - fee: SignedAmount::from_sat(tx.fee.unwrap_or_default() as i64), + hash, + fee: SignedAmount::from_sat(tx.fee as i64), }) } pub async fn get_utxos_for_address(&self, address: &Address) -> Result, Error> { - let utxos: Vec = self.get_and_decode(&format!("/address/{address}/utxo")).await?; + let utxos: Vec = self.get_and_decode(&format!("/address/{address}/utxo")).await?; // NOTE: includes unconfirmed mempool txs utxos .into_iter() .map(|utxo| { Ok(Utxo { outpoint: OutPoint { - txid: Txid::from_hex(&utxo.txid)?, - vout: utxo.vout as u32, + txid: utxo.txid, + vout: utxo.vout, }, - value: utxo.value as u64, + value: utxo.value, }) }) .collect::, Error>>() } pub(crate) async fn get_script_pubkey(&self, outpoint: OutPoint) -> Result { - let tx: ElectrsTransaction = self + let tx: TransactionValue = self .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) .await?; - Ok(Script::from_str( - &tx.vout - .ok_or(Error::NoPrevOut)? - .get(outpoint.vout as usize) - .ok_or(Error::NoPrevOut)? - .clone() - .scriptpubkey - .ok_or(Error::NoPrevOut)?, - )?) + Ok(tx + .vout + .get(outpoint.vout as usize) + .ok_or(Error::NoPrevOut)? + .scriptpubkey + .clone()) } pub(crate) async fn get_prev_value(&self, outpoint: &OutPoint) -> Result { - let tx: ElectrsTransaction = self + let tx: TransactionValue = self .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) .await?; Ok(tx .vout - .ok_or(Error::NoPrevOut)? .get(outpoint.vout as usize) .ok_or(Error::NoPrevOut)? .clone() - .value - .map(|v| v as u64) - .ok_or(Error::NoPrevOut)?) + .value) } // TODO: modify upstream to return a human-readable error @@ -232,6 +225,28 @@ impl ElectrsClient { Ok(Txid::from_str(&txid)?) } + pub(crate) async fn get_txs_by_scripthash(&self, script_hash: Vec) -> Result, Error> { + let mut last_seen_txid = Default::default(); + let mut txs = Vec::::new(); + loop { + // NOTE: includes unconfirmed txs + let mut transactions: Vec = self + .get_and_decode(&format!( + "/scripthash/{scripthash}/txs/chain/{last_seen_txid}", + scripthash = script_hash.to_hex() + )) + .await?; + let page_size = transactions.len(); + last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.to_string()); + txs.append(&mut transactions); + if page_size < ELECTRS_TRANSACTIONS_PER_PAGE { + // no further pages + break; + } + } + Ok(txs) + } + pub(crate) async fn get_tx_for_op_return( &self, address: Address, @@ -249,35 +264,17 @@ impl ElectrsClient { hasher.result().as_slice().to_vec() }; - let mut last_seen_txid = Default::default(); - let mut txs = Vec::::new(); - loop { - // NOTE: includes unconfirmed txs - let mut transactions: Vec = self - .get_and_decode(&format!( - "/scripthash/{scripthash}/txs/chain/{last_seen_txid}", - scripthash = script_hash.to_hex() - )) - .await?; - let page_size = transactions.len(); - last_seen_txid = transactions.last().map_or(Default::default(), |tx| tx.txid.clone()); - txs.append(&mut transactions); - if page_size < ELECTRS_TRANSACTIONS_PER_PAGE { - // no further pages - break; - } - } + let txs = self.get_txs_by_scripthash(script_hash).await?; log::info!("Found {} transactions", txs.len()); let address = address.to_string(); for tx in txs { let largest = tx .vout - .unwrap_or_default() .iter() .filter_map(|vout| { if vout.scriptpubkey_address.contains(&address) { - Some(vout.value.unwrap_or_default() as u64) + Some(vout.value) } else { None } @@ -286,8 +283,7 @@ impl ElectrsClient { .unwrap_or_default(); if largest as u128 >= amount { - let txid = Txid::from_str(&tx.txid)?; - return Ok(Some(txid)); + return Ok(Some(tx.txid)); } } Ok(None) @@ -299,22 +295,19 @@ mod tests { use super::*; use bitcoincore_rpc::bitcoin::hashes::{hex::FromHex, sha256::Hash as Sha256Hash, Hash}; - use esplora_btc_api::apis::configuration::Configuration as ElectrsConfiguration; // TODO: mock the electrs endpoint async fn test_electrs(url: &str, script_hex: &str, expected_txid: &str) { - let config = ElectrsConfiguration { - base_path: url.to_owned(), - ..Default::default() - }; - let script_bytes = Vec::from_hex(script_hex).unwrap(); let script_hash = Sha256Hash::hash(&script_bytes); + let expected_txid = Txid::from_hex(expected_txid).unwrap(); - let txs = esplora_btc_api::apis::scripthash_api::get_txs_by_scripthash(&config, &hex::encode(script_hash)) + let electrs_client = ElectrsClient::new(Some(url.to_owned()), Network::Bitcoin).unwrap(); + let txs = electrs_client + .get_txs_by_scripthash(script_hash.to_vec()) .await .unwrap(); - assert!(txs.iter().any(|tx| { &tx.txid == expected_txid })); + assert!(txs.iter().any(|tx| tx.txid.eq(&expected_txid))); } #[tokio::test(flavor = "multi_thread")] diff --git a/bitcoin/src/electrs/types.rs b/bitcoin/src/electrs/types.rs new file mode 100644 index 000000000..aa81be13d --- /dev/null +++ b/bitcoin/src/electrs/types.rs @@ -0,0 +1,67 @@ +use crate::{BlockHash, Script, Txid}; +use serde::Deserialize; + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/util/transaction.rs#L17-L26 +#[derive(Deserialize)] +pub struct TransactionStatus { + pub confirmed: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub block_height: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub block_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub block_time: Option, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L167-L189 +#[derive(Deserialize)] +pub struct TxInValue { + pub txid: Txid, + pub vout: u32, + pub prevout: Option, + pub scriptsig: Script, + pub scriptsig_asm: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub witness: Option>, + pub is_coinbase: bool, + pub sequence: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub inner_redeemscript_asm: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub inner_witnessscript_asm: Option, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L239-L270 +#[derive(Deserialize)] +pub struct TxOutValue { + pub scriptpubkey: Script, + pub scriptpubkey_asm: String, + pub scriptpubkey_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub scriptpubkey_address: Option, + pub value: u64, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L115-L127 +#[derive(Deserialize)] +pub struct TransactionValue { + pub txid: Txid, + pub version: u32, + pub locktime: u32, + pub vin: Vec, + pub vout: Vec, + pub size: u32, + pub weight: u32, + pub fee: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +// https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L356-L396 +#[derive(Deserialize)] +pub struct UtxoValue { + pub txid: Txid, + pub vout: u32, + pub status: TransactionStatus, + pub value: u64, +} diff --git a/bitcoin/src/lib.rs b/bitcoin/src/lib.rs index 0ca430ee8..cba435726 100644 --- a/bitcoin/src/lib.rs +++ b/bitcoin/src/lib.rs @@ -1019,8 +1019,6 @@ impl BitcoinCoreApi for BitcoinCore { } }; tx.vout - .as_ref() - .unwrap_or(&vec![]) .iter() .any(|output| matches!(&output.scriptpubkey_address, Some(addr) if addr == &address)) }); From 34d5e55a949ede3c1b1f018caf6c752298047eb7 Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 29 Mar 2023 13:05:43 +0000 Subject: [PATCH 07/13] chore: fix clippy warnings Signed-off-by: Gregory Hill --- Cargo.lock | 10 +++++----- bitcoin/src/electrs/mod.rs | 7 +------ bitcoin/src/light/mod.rs | 4 ++-- bitcoin/src/light/wallet.rs | 8 ++------ 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05db2d8d3..6621160d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15970,12 +15970,12 @@ dependencies = [ "pkg-config", ] -[[patch.unused]] -name = "sp-serializer" -version = "4.0.0-dev" -source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" - [[patch.unused]] name = "orml-xcm" version = "0.4.1-dev" source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" + +[[patch.unused]] +name = "sp-serializer" +version = "4.0.0-dev" +source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index 1f873e619..faa39c38f 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -201,12 +201,7 @@ impl ElectrsClient { let tx: TransactionValue = self .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) .await?; - Ok(tx - .vout - .get(outpoint.vout as usize) - .ok_or(Error::NoPrevOut)? - .clone() - .value) + Ok(tx.vout.get(outpoint.vout as usize).ok_or(Error::NoPrevOut)?.value) } // TODO: modify upstream to return a human-readable error diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 131655b0c..c031ff4ba 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -28,7 +28,7 @@ impl BitcoinLight { let electrs_client = ElectrsClient::new(electrs_url, network)?; let wallet = wallet::Wallet::new(network, electrs_client.clone()); // store the derivation key so it can be used for change - wallet.put_p2wpkh_key(private_key.inner.clone())?; + wallet.put_p2wpkh_key(private_key.inner)?; Ok(Self { private_key, secp_ctx: secp256k1::Secp256k1::new(), @@ -269,7 +269,7 @@ impl BitcoinCoreApi for BitcoinLight { existing_transaction, return_to_self.unwrap(), fee_rate, - Some(txid.clone()), + Some(*txid), ) .await?; let txid = self.send_transaction(tx).await?; diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index 2c71748bd..a5d0a0597 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -211,7 +211,7 @@ fn get_dust_threshold(tx_out: &TxOut, dust_relay_fee_in: &FeeRate) -> u64 { n_size += 32 + 4 + 1 + 107 + 4; } - return dust_relay_fee_in.get_fee(n_size); + dust_relay_fee_in.get_fee(n_size) } pub type KeyStore = Arc>>; @@ -255,11 +255,7 @@ pub fn available_coins( match state.electrs.get_utxos_for_address(&address).await { Ok(utxos) => { state.utxos = VecDeque::from(utxos); - if let Some(utxo) = state.utxos.pop_front() { - Some((Ok(utxo), state)) - } else { - None - } + state.utxos.pop_front().map(|utxo| (Ok(utxo), state)) } Err(e) => Some((Err(Error::ElectrsError(e)), state)), } From 4cad56b3c06276cf0431bc3ef2259501312596ae Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 29 Mar 2023 13:07:55 +0000 Subject: [PATCH 08/13] chore: remove outdated todo note Signed-off-by: Gregory Hill --- Cargo.lock | 10 +++++----- bitcoin/src/light/mod.rs | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6621160d6..05db2d8d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15970,12 +15970,12 @@ dependencies = [ "pkg-config", ] -[[patch.unused]] -name = "orml-xcm" -version = "0.4.1-dev" -source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" - [[patch.unused]] name = "sp-serializer" version = "4.0.0-dev" source = "git+https://github.com/paritytech//substrate?branch=polkadot-v0.9.37#f38bd6671d460293c93062cc1e4fe9e9e490cb29" + +[[patch.unused]] +name = "orml-xcm" +version = "0.4.1-dev" +source = "git+https://github.com/open-web3-stack//open-runtime-module-library?rev=24f0a8b6e04e1078f70d0437fb816337cdf4f64c#24f0a8b6e04e1078f70d0437fb816337cdf4f64c" diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index c031ff4ba..22e4c66c6 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -78,8 +78,6 @@ impl BitcoinLight { .await } - // TODO: hold tx lock until inclusion otherwise electrs may report stale utxos - // need to check when electrs knows that a utxo has been spent async fn send_transaction(&self, transaction: LockedTransaction) -> Result { let txid = self.electrs.send_transaction(transaction.transaction).await?; Ok(txid) From b2560224cc5aacf6ca1a9e77718a948b8284787f Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 29 Mar 2023 13:49:25 +0000 Subject: [PATCH 09/13] chore: add get_tx_for_op_return to mocks Signed-off-by: Gregory Hill --- runtime/src/integration/bitcoin_simulator.rs | 7 ++++++- vault/src/execution.rs | 1 + vault/src/metrics.rs | 1 + vault/src/replace.rs | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/runtime/src/integration/bitcoin_simulator.rs b/runtime/src/integration/bitcoin_simulator.rs index 705161880..390cd766e 100644 --- a/runtime/src/integration/bitcoin_simulator.rs +++ b/runtime/src/integration/bitcoin_simulator.rs @@ -509,7 +509,12 @@ impl BitcoinCoreApi for MockBitcoinCore { unimplemented!() } - async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, Error> { + async fn get_tx_for_op_return( + &self, + address: Address, + amount: u128, + data: H256, + ) -> Result, BitcoinError> { unimplemented!() } } diff --git a/vault/src/execution.rs b/vault/src/execution.rs index 169f022b0..1c97213d3 100644 --- a/vault/src/execution.rs +++ b/vault/src/execution.rs @@ -823,6 +823,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, BitcoinError>; } } diff --git a/vault/src/metrics.rs b/vault/src/metrics.rs index 831293e05..2840874aa 100644 --- a/vault/src/metrics.rs +++ b/vault/src/metrics.rs @@ -825,6 +825,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, BitcoinError>; } } diff --git a/vault/src/replace.rs b/vault/src/replace.rs index 214ac4e6e..816d664ce 100644 --- a/vault/src/replace.rs +++ b/vault/src/replace.rs @@ -291,6 +291,7 @@ mod tests { ) -> Result; async fn is_in_mempool(&self, txid: Txid) -> Result; async fn fee_rate(&self, txid: Txid) -> Result; + async fn get_tx_for_op_return(&self, address: Address, amount: u128, data: H256) -> Result, BitcoinError>; } } From 336899f9cbcc139f3184b7223a44bf3df84b111f Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 29 Mar 2023 15:11:08 +0000 Subject: [PATCH 10/13] chore: update bitcoin user / pass in workflow Signed-off-by: Gregory Hill --- .github/workflows/cargo-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index d51233e60..8a01ef8be 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -55,8 +55,8 @@ jobs: env: RUST_LOG: info,regalloc=warn BITCOIN_RPC_URL: http://127.0.0.1:18443 - BITCOIN_RPC_USER: user - BITCOIN_RPC_PASS: pass + BITCOIN_RPC_USER: rpcuser + BITCOIN_RPC_PASS: rpcpassword ELECTRS_URL: http://localhost:3002 run: | docker-compose up --detach From b59a97adc5c1659a60e4be197c1f93e626ac55a0 Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Thu, 30 Mar 2023 06:21:56 +0000 Subject: [PATCH 11/13] chore: bitcoin-sim should return none for op-return Signed-off-by: Gregory Hill --- runtime/src/integration/bitcoin_simulator.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/src/integration/bitcoin_simulator.rs b/runtime/src/integration/bitcoin_simulator.rs index 390cd766e..7afebece4 100644 --- a/runtime/src/integration/bitcoin_simulator.rs +++ b/runtime/src/integration/bitcoin_simulator.rs @@ -511,10 +511,10 @@ impl BitcoinCoreApi for MockBitcoinCore { async fn get_tx_for_op_return( &self, - address: Address, - amount: u128, - data: H256, + _address: Address, + _amount: u128, + _data: H256, ) -> Result, BitcoinError> { - unimplemented!() + Ok(None) } } From 114791e8ce9c96d27e40cde6a04cdc7bcccaabe3 Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 12 Apr 2023 15:07:49 +0100 Subject: [PATCH 12/13] refactor: fallible usize conversion Signed-off-by: Gregory Hill --- bitcoin/src/electrs/mod.rs | 10 +++++++--- bitcoin/src/light/mod.rs | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index faa39c38f..db364a59c 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -11,7 +11,7 @@ use crate::{ use futures::future::{join_all, try_join}; use reqwest::{Client, Url}; use sha2::{Digest, Sha256}; -use std::str::FromStr; +use std::{convert::TryFrom, str::FromStr}; // https://github.com/Blockstream/electrs/blob/adedee15f1fe460398a7045b292604df2161adc0/src/rest.rs#L42 const ELECTRS_TRANSACTIONS_PER_PAGE: usize = 25; @@ -191,7 +191,7 @@ impl ElectrsClient { .await?; Ok(tx .vout - .get(outpoint.vout as usize) + .get(usize::try_from(outpoint.vout)?) .ok_or(Error::NoPrevOut)? .scriptpubkey .clone()) @@ -201,7 +201,11 @@ impl ElectrsClient { let tx: TransactionValue = self .get_and_decode(&format!("/tx/{txid}", txid = outpoint.txid)) .await?; - Ok(tx.vout.get(outpoint.vout as usize).ok_or(Error::NoPrevOut)?.value) + Ok(tx + .vout + .get(usize::try_from(outpoint.vout)?) + .ok_or(Error::NoPrevOut)? + .value) } // TODO: modify upstream to return a human-readable error diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 22e4c66c6..38fa0c6fd 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -8,7 +8,7 @@ pub use error::Error; use async_trait::async_trait; use backoff::future::retry; use futures::future::{join_all, try_join, try_join_all}; -use std::{sync::Arc, time::Duration}; +use std::{convert::TryFrom, sync::Arc, time::Duration}; use tokio::{sync::Mutex, time::sleep}; const RETRY_DURATION: Duration = Duration::from_millis(1000); @@ -335,7 +335,7 @@ impl BitcoinCoreApi for BitcoinLight { let prev_tx = self.get_transaction(&input.previous_output.txid, None).await?; let prev_out = prev_tx .output - .get(input.previous_output.vout as usize) + .get(usize::try_from(input.previous_output.vout)?) .ok_or(Error::NoPrevOut)?; Ok::(prev_out.value) })) From 5bd1968821af666670669aacf0d0eff02896d9d4 Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Wed, 12 Apr 2023 15:29:05 +0100 Subject: [PATCH 13/13] chore: improve doc comments Signed-off-by: Gregory Hill --- bitcoin/src/electrs/mod.rs | 3 +++ bitcoin/src/light/mod.rs | 1 + bitcoin/src/light/wallet.rs | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bitcoin/src/electrs/mod.rs b/bitcoin/src/electrs/mod.rs index db364a59c..8acfd707e 100644 --- a/bitcoin/src/electrs/mod.rs +++ b/bitcoin/src/electrs/mod.rs @@ -246,6 +246,9 @@ impl ElectrsClient { Ok(txs) } + /// Returns the *largest* payment to the `address` which is + /// greater than or equal to the specified `amount` and contains + /// an `OP_RETURN` output with `data`. pub(crate) async fn get_tx_for_op_return( &self, address: Address, diff --git a/bitcoin/src/light/mod.rs b/bitcoin/src/light/mod.rs index 38fa0c6fd..e1e29e4f9 100644 --- a/bitcoin/src/light/mod.rs +++ b/bitcoin/src/light/mod.rs @@ -257,6 +257,7 @@ impl BitcoinCoreApi for BitcoinLight { } }); + // clear the witnesses for fee estimation existing_transaction .input .iter_mut() diff --git a/bitcoin/src/light/wallet.rs b/bitcoin/src/light/wallet.rs index a5d0a0597..d02a5ef0f 100644 --- a/bitcoin/src/light/wallet.rs +++ b/bitcoin/src/light/wallet.rs @@ -353,7 +353,7 @@ impl Wallet { log::info!("Found utxo: {}", utxo.outpoint.txid); if prev_txid.contains(&utxo.outpoint.txid) { - // skip if trying to spend previous tx (RBF) + // skip if trying to spend from the tx we are replacing continue; }