From 7a79407f6e6f77dfc3d9e09dda4768ac75c9402e Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:21:25 +0400 Subject: [PATCH 01/25] feat: add `silent` kwarg for tx.replace --- brownie/network/transaction.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/brownie/network/transaction.py b/brownie/network/transaction.py index e73df5e6a..b197379c4 100644 --- a/brownie/network/transaction.py +++ b/brownie/network/transaction.py @@ -262,7 +262,10 @@ def confirmations(self) -> int: return web3.eth.blockNumber - self.block_number + 1 def replace( - self, increment: Optional[float] = None, gas_price: Optional[Wei] = None, + self, + increment: Optional[float] = None, + gas_price: Optional[Wei] = None, + silent: Optional[bool] = None, ) -> "TransactionReceipt": """ Rebroadcast this transaction with a higher gas price. @@ -274,8 +277,10 @@ def replace( increment : float, optional Multiplier applied to the gas price of this transaction in order to determine the new gas price - gas_price: Wei, optional + gas_price : Wei, optional Absolute gas price to use in the replacement transaction + silent : bool, optional + Toggle console verbosity (default is same setting as this transaction) Returns ------- @@ -292,6 +297,9 @@ def replace( if increment is not None: gas_price = Wei(self.gas_price * 1.1) + if silent is None: + silent = self._silent + return self.sender.transfer( # type: ignore self.receiver, self.value, @@ -300,7 +308,7 @@ def replace( data=self.input, nonce=self.nonce, required_confs=0, - silent=self._silent, + silent=silent, ) def wait(self, required_confs: int) -> None: From 04386bcc9d64f9973eb6df1a58e93aed38ddf944 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:23:44 +0400 Subject: [PATCH 02/25] feat: brownie.network.gas --- brownie/network/gas/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 brownie/network/gas/__init__.py diff --git a/brownie/network/gas/__init__.py b/brownie/network/gas/__init__.py new file mode 100644 index 000000000..e69de29bb From c6997ad92e73bb24e4505589bc1ee43acb12e755 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:27:44 +0400 Subject: [PATCH 03/25] feat: gas strategy abc's --- brownie/network/gas/bases.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 brownie/network/gas/bases.py diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py new file mode 100644 index 000000000..afbb071ce --- /dev/null +++ b/brownie/network/gas/bases.py @@ -0,0 +1,36 @@ +from abc import ABC, abstractmethod +from typing import Optional + + +class GasABC(ABC): + pass + + +class SimpleGasStrategy(GasABC): + @abstractmethod + def get_gas_price(self) -> int: + raise NotImplementedError + + +class BlockGasStrategy(GasABC): + + block_duration = 2 + + def __init__(self, block_duration: int = 2): + self.block_duration = block_duration + + @abstractmethod + def get_gas_price(self, current_gas_price: int, elapsed_blocks: int) -> Optional[int]: + raise NotImplementedError + + +class TimeGasStrategy(GasABC): + + time_duration = 30 + + def __init__(self, time_duration: int = 30) -> None: + self.time_duration = time_duration + + @abstractmethod + def get_gas_price(self, current_gas_price: int, elapsed_time: int) -> Optional[int]: + raise NotImplementedError From aea7e11660d9e8647f59e38e7a075dd1f7da863e Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:29:43 +0400 Subject: [PATCH 04/25] feat: gas strategy queue and loop --- brownie/network/gas/bases.py | 55 ++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index afbb071ce..484812ecc 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -1,9 +1,20 @@ +import threading +import time from abc import ABC, abstractmethod -from typing import Optional +from collections import deque +from typing import Any, Optional + +from brownie.network.web3 import web3 class GasABC(ABC): - pass + def _add_tx(self, txreceipt: Any) -> None: + if isinstance(self, SimpleGasStrategy): + return + + number = web3.eth.blockNumber if isinstance(self, BlockGasStrategy) else time.time() + _queue.append((self, txreceipt, number, number)) + _event.set() class SimpleGasStrategy(GasABC): @@ -34,3 +45,43 @@ def __init__(self, time_duration: int = 30) -> None: @abstractmethod def get_gas_price(self, current_gas_price: int, elapsed_time: int) -> Optional[int]: raise NotImplementedError + + +def _update_loop(): + while True: + if not _queue: + _event.wait() + _event.clear() + + try: + gas_strategy, tx, initial, latest = _queue.popleft() + except IndexError: + continue + + if tx.status >= 0: + continue + + if isinstance(gas_strategy, BlockGasStrategy): + height = web3.eth.blockNumber + if height - latest >= gas_strategy.block_duration: + gas_price = gas_strategy.get_gas_price(tx.gas_price, height - initial) + if gas_price is not None: + tx = tx.replace(gas_price=gas_price, silent=True) + latest = web3.eth.blockNumber + + else: + if time.time() - latest >= gas_strategy.time_duration: + gas_price = gas_strategy.get_gas_price(tx.gas_price, time.time() - initial) + if gas_price is not None: + tx = tx.replace(gas_price=gas_price, silent=True) + latest = time.time() + + _queue.append((gas_strategy, tx, initial, latest)) + time.sleep(1) + + +_queue: deque = deque() +_event = threading.Event() + +_repricing_thread = threading.Thread(target=_update_loop, daemon=True) +_repricing_thread.start() From dd6b2b024eb69b4f2b5d2117eca96747bf287efd Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:37:56 +0400 Subject: [PATCH 05/25] feat: implement gas strategy within account transaction logic --- brownie/network/account.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/brownie/network/account.py b/brownie/network/account.py index 646127be9..c46370eed 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -24,6 +24,7 @@ ) from brownie.utils import color +from .gas.bases import GasABC from .rpc import Rpc from .state import Chain, TxHistory, _revert_register from .transaction import TransactionReceipt @@ -368,11 +369,19 @@ def _gas_limit( return Wei(gas_limit) - def _gas_price(self) -> Wei: - gas_price = CONFIG.active_network["settings"]["gas_price"] + def _gas_price(self, gas_price: Any = None) -> Tuple[Wei, Optional[GasABC]]: + if gas_price is None: + gas_price = CONFIG.active_network["settings"]["gas_price"] + + if isinstance(gas_price, GasABC): + if gas_price.get_gas_price.__code__.co_argcount: # type: ignore + return Wei(gas_price.get_gas_price(None, 0)), gas_price # type: ignore + else: + return Wei(gas_price.get_gas_price()), None # type: ignore + if isinstance(gas_price, bool) or gas_price in (None, "auto"): - return web3.eth.generateGasPrice() - return Wei(gas_price) + return web3.eth.generateGasPrice(), None + return Wei(gas_price), None def _check_for_revert(self, tx: Dict) -> None: try: @@ -428,7 +437,7 @@ def deploy( silent = bool(CONFIG.mode == "test" or CONFIG.argv["silent"]) with self._lock: try: - gas_price = Wei(gas_price) if gas_price is not None else self._gas_price() + gas_price, gas_strategy = self._gas_price(gas_price) gas_limit = Wei(gas_limit) or self._gas_limit( None, amount, gas_price, gas_buffer, data ) @@ -579,7 +588,7 @@ def transfer( if silent is None: silent = bool(CONFIG.mode == "test" or CONFIG.argv["silent"]) with self._lock: - gas_price = Wei(gas_price) if gas_price is not None else self._gas_price() + gas_price, gas_strategy = self._gas_price(gas_price) gas_limit = Wei(gas_limit) or self._gas_limit(to, amount, gas_price, gas_buffer, data) tx = { "from": self.address, @@ -612,6 +621,10 @@ def transfer( # add the TxHistory before waiting for confirmation, this way the tx # object is available if the user CTRL-C to stop waiting in the console history._add_tx(receipt) + + if gas_strategy is not None: + gas_strategy._add_tx(receipt) + if required_confs > 0: receipt._confirmed.wait() From 4df56d005cb2920c13d4fd34ff464039144f6cf2 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:39:12 +0400 Subject: [PATCH 06/25] feat: gasnow strategy --- brownie/network/gas/strategies.py | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 brownie/network/gas/strategies.py diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py new file mode 100644 index 000000000..d3d86e093 --- /dev/null +++ b/brownie/network/gas/strategies.py @@ -0,0 +1,59 @@ +import threading +import time + +import requests + +from .bases import BlockGasStrategy, SimpleGasStrategy + +_gasnow = {"time": 0, "data": None} +_gasnow_lock = threading.Lock() + + +def _fetch_gasnow(key): + with _gasnow_lock: + if time.time() - _gasnow["time"] > 15: + data = None + for i in range(12): + response = requests.get( + "https://www.gasnow.org/api/v3/gas/price?utm_source=brownie" + ) + if response.status_code != 200: + time.sleep(5) + continue + data = response.json()["data"] + if data is None: + raise ValueError + _gasnow["time"] = data.pop("timestamp") // 1000 + _gasnow["data"] = data + + return _gasnow["data"][key] + + +class GasNowStrategy(SimpleGasStrategy): + def __init__(self, speed: str = "fast"): + if speed not in ("rapid", "fast", "standard", "slow"): + raise ValueError("`speed` must be one of: rapid, fast, standard, slow") + self.speed = speed + + def get_gas_price(self): + return _fetch_gasnow(self.speed) + + +class GasNowScalingStrategy(BlockGasStrategy): + def __init__( + self, initial_speed: str = "standard", increment: float = 1.1, block_duration: int = 2 + ): + super().__init__(block_duration) + if initial_speed not in ("rapid", "fast", "standard", "slow"): + raise ValueError("`initial_speed` must be one of: rapid, fast, standard, slow") + self.speed = initial_speed + self.increment = increment + + def get_gas_price(self, current_gas_price, elapsed_blocks): + if current_gas_price is None: + return _fetch_gasnow(self.speed) + rapid_gas_price = _fetch_gasnow("rapid") + new_gas_price = max(int(current_gas_price * self.increment), _fetch_gasnow(self.speed)) + if new_gas_price <= rapid_gas_price: + return new_gas_price + return None From 8caaf1a7fd41a560bfde16e892cab48a33124f2c Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 04:55:41 +0400 Subject: [PATCH 07/25] fix: handle ValueError in replacement loop --- brownie/network/gas/bases.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index 484812ecc..ddce5ea11 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -66,15 +66,21 @@ def _update_loop(): if height - latest >= gas_strategy.block_duration: gas_price = gas_strategy.get_gas_price(tx.gas_price, height - initial) if gas_price is not None: - tx = tx.replace(gas_price=gas_price, silent=True) - latest = web3.eth.blockNumber + try: + tx = tx.replace(gas_price=gas_price, silent=True) + latest = web3.eth.blockNumber + except ValueError: + pass else: if time.time() - latest >= gas_strategy.time_duration: gas_price = gas_strategy.get_gas_price(tx.gas_price, time.time() - initial) if gas_price is not None: - tx = tx.replace(gas_price=gas_price, silent=True) - latest = time.time() + try: + tx = tx.replace(gas_price=gas_price, silent=True) + latest = time.time() + except ValueError: + pass _queue.append((gas_strategy, tx, initial, latest)) time.sleep(1) From 9c849d72e2d814c29557c4934012243634dfc55f Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 17:38:08 +0400 Subject: [PATCH 08/25] feat: return confirmed when blocking tx drops --- brownie/exceptions.py | 4 ++++ brownie/network/account.py | 40 +++++++++++++++++++++++----------- brownie/network/transaction.py | 3 ++- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/brownie/exceptions.py b/brownie/exceptions.py index e00eeb6cf..1c4d29093 100644 --- a/brownie/exceptions.py +++ b/brownie/exceptions.py @@ -131,6 +131,10 @@ def _with_attr(self, **kwargs) -> "VirtualMachineError": return self +class TransactionError(Exception): + pass + + class EventLookupError(LookupError): pass diff --git a/brownie/network/account.py b/brownie/network/account.py index c46370eed..ba040c5af 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -19,6 +19,7 @@ from brownie.exceptions import ( ContractNotFound, IncompatibleEVMVersion, + TransactionError, UnknownAccount, VirtualMachineError, ) @@ -469,11 +470,12 @@ def deploy( name=contract._name + ".constructor", revert_data=revert_data, ) - # add the TxHistory before waiting for confirmation, this way the tx - # object is available if the user CTRL-C to stop waiting in the console - history._add_tx(receipt) - if required_confs > 0: - receipt._confirmed.wait() + + # add the TxHistory before waiting for confirmation, this way the tx + # object is available if the user CTRL-C to stop waiting in the console + history._add_tx(receipt) + if required_confs > 0: + receipt._confirmed.wait() add_thread = threading.Thread(target=contract._add_from_tx, args=(receipt,), daemon=True) add_thread.start() @@ -618,15 +620,26 @@ def transfer( silent=silent, revert_data=revert_data, ) - # add the TxHistory before waiting for confirmation, this way the tx - # object is available if the user CTRL-C to stop waiting in the console - history._add_tx(receipt) - - if gas_strategy is not None: - gas_strategy._add_tx(receipt) - if required_confs > 0: - receipt._confirmed.wait() + # add the TxHistory before waiting for confirmation, this way the tx + # object is available if the user CTRL-C to stop waiting in the console + history._add_tx(receipt) + + if gas_strategy is not None: + gas_strategy._add_tx(receipt) + + if required_confs > 0: + receipt._confirmed.wait() + if receipt.status == -2: + # if transaction was dropped (status -2), find and return the tx that confirmed + receipt = next( + history.filter(sender=self, nonce=receipt.nonce, key=lambda k: k.status >= 0), + None, + ) # type: ignore + if receipt is None: + raise TransactionError( + f"Transaction was dropped without a known replacement: {txid}" + ) if rpc.is_active(): undo_thread = threading.Thread( @@ -640,6 +653,7 @@ def transfer( daemon=True, ) undo_thread.start() + receipt._raise_if_reverted(exc) return receipt diff --git a/brownie/network/transaction.py b/brownie/network/transaction.py index b197379c4..7b811b427 100644 --- a/brownie/network/transaction.py +++ b/brownie/network/transaction.py @@ -327,7 +327,7 @@ def wait(self, required_confs: int) -> None: sender_nonce = web3.eth.getTransactionCount(str(self.sender)) if sender_nonce > self.nonce: self.status = Status(-2) - print("This transaction was replaced.") + self._confirmed.set() return time.sleep(1) @@ -412,6 +412,7 @@ def _await_confirmation(self, tx: Dict, required_confs: int = 1) -> None: if expect_confirmed: # if we expected confirmation based on the nonce, tx likely dropped self.status = Status(-2) + self._confirmed.set() return self.block_number = receipt["blockNumber"] From 3b0e5b75bd48bca9b7678c018215bc7ad4d9361b Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 20:06:01 +0400 Subject: [PATCH 09/25] refactor: handle dropping tx's in transaction instead of history --- brownie/network/state.py | 8 -------- brownie/network/transaction.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/brownie/network/state.py b/brownie/network/state.py index 3f433c1d8..f92ed08f5 100644 --- a/brownie/network/state.py +++ b/brownie/network/state.py @@ -75,14 +75,6 @@ def _revert(self, height: int) -> None: def _add_tx(self, tx: TransactionReceipt) -> None: self._list.append(tx) - confirm_thread = threading.Thread(target=self._await_confirm, args=(tx,), daemon=True) - confirm_thread.start() - - def _await_confirm(self, tx: TransactionReceipt) -> None: - # in case of multiple tx's with the same nonce, remove the dropped tx's upon confirmation - tx._confirmed.wait() - for dropped_tx in self.filter(sender=tx.sender, nonce=tx.nonce, key=lambda k: k != tx): - self._list.remove(dropped_tx) def clear(self) -> None: self._list.clear() diff --git a/brownie/network/transaction.py b/brownie/network/transaction.py index 7b811b427..f0703ed0f 100644 --- a/brownie/network/transaction.py +++ b/brownie/network/transaction.py @@ -150,9 +150,12 @@ def __init__( if not self._silent: print(f"Transaction sent: {color('bright blue')}{txid}{color}") + # this event is set once the transaction is confirmed or dropped + # it is used to waiting during blocking transaction actions + self._confirmed = threading.Event() + # internal attributes self._call_cost = 0 - self._confirmed = threading.Event() self._trace_exc: Optional[Exception] = None self._trace_origin: Optional[str] = None self._raw_trace: Optional[List] = None @@ -453,7 +456,14 @@ def _await_confirmation(self, tx: Dict, required_confs: int = 1) -> None: self._expand_trace() if not self._silent and required_confs > 0: print(self._confirm_output()) + + # set the confirmation event and mark other tx's with the same nonce as dropped self._confirmed.set() + for dropped_tx in state.TxHistory().filter( + sender=self.sender, nonce=self.nonce, key=lambda k: k != self + ): + dropped_tx.status = Status(-2) + dropped_tx._confirmed.set() def _set_from_tx(self, tx: Dict) -> None: if not self.sender: From 98decb01f2d56f26df561d4152a590c4a557c115 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Tue, 17 Nov 2020 20:11:39 +0400 Subject: [PATCH 10/25] feat: ensure returned tx is the confirmed one --- brownie/network/account.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/brownie/network/account.py b/brownie/network/account.py index ba040c5af..a708ec063 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -632,14 +632,20 @@ def transfer( receipt._confirmed.wait() if receipt.status == -2: # if transaction was dropped (status -2), find and return the tx that confirmed - receipt = next( - history.filter(sender=self, nonce=receipt.nonce, key=lambda k: k.status >= 0), - None, - ) # type: ignore - if receipt is None: - raise TransactionError( - f"Transaction was dropped without a known replacement: {txid}" - ) + replacements = history.filter( + sender=self, nonce=receipt.nonce, key=lambda k: k.status != -2 + ) + while True: + if not replacements: + raise TransactionError( + f"Transaction was dropped without a known replacement: {txid}" + ) + if len(replacements) == 1: + receipt = replacements[0] + break + # in case we have multiple tx's where the status is still unresolved + replacements = [i for i in replacements if i.status != 2] + time.sleep(0.5) if rpc.is_active(): undo_thread = threading.Thread( From e0fc0ce33b67bf8553e1e32cb2659a0228000875 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Wed, 18 Nov 2020 14:47:42 +0400 Subject: [PATCH 11/25] feat: allow use of gas strategy in network.gas_price --- brownie/network/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/brownie/network/main.py b/brownie/network/main.py index 9c58d3a4f..b7ca4b853 100644 --- a/brownie/network/main.py +++ b/brownie/network/main.py @@ -9,6 +9,7 @@ from brownie.exceptions import BrownieEnvironmentWarning from .account import Accounts +from .gas.bases import GasABC from .rpc import Rpc from .state import Chain, _notify_registry from .web3 import web3 @@ -114,7 +115,9 @@ def gas_price(*args: Tuple[Union[int, str, bool, None]]) -> Union[int, bool]: if not is_connected(): raise ConnectionError("Not connected to any network") if args: - if args[0] in (None, False, True, "auto"): + if isinstance(args[0], GasABC): + CONFIG.active_network["settings"]["gas_price"] = args[0] + elif args[0] in (None, False, True, "auto"): CONFIG.active_network["settings"]["gas_price"] = False else: try: From 8659433e7d59646902e2b1725ac21639c27e9b5a Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Wed, 18 Nov 2020 15:16:14 +0400 Subject: [PATCH 12/25] fix: include gas strategy on deployment tx's --- brownie/network/account.py | 63 ++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/brownie/network/account.py b/brownie/network/account.py index a708ec063..32f958f06 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -471,11 +471,7 @@ def deploy( revert_data=revert_data, ) - # add the TxHistory before waiting for confirmation, this way the tx - # object is available if the user CTRL-C to stop waiting in the console - history._add_tx(receipt) - if required_confs > 0: - receipt._confirmed.wait() + receipt = self._await_confirmation(receipt, gas_strategy, required_confs) add_thread = threading.Thread(target=contract._add_from_tx, args=(receipt,), daemon=True) add_thread.start() @@ -621,31 +617,7 @@ def transfer( revert_data=revert_data, ) - # add the TxHistory before waiting for confirmation, this way the tx - # object is available if the user CTRL-C to stop waiting in the console - history._add_tx(receipt) - - if gas_strategy is not None: - gas_strategy._add_tx(receipt) - - if required_confs > 0: - receipt._confirmed.wait() - if receipt.status == -2: - # if transaction was dropped (status -2), find and return the tx that confirmed - replacements = history.filter( - sender=self, nonce=receipt.nonce, key=lambda k: k.status != -2 - ) - while True: - if not replacements: - raise TransactionError( - f"Transaction was dropped without a known replacement: {txid}" - ) - if len(replacements) == 1: - receipt = replacements[0] - break - # in case we have multiple tx's where the status is still unresolved - replacements = [i for i in replacements if i.status != 2] - time.sleep(0.5) + receipt = self._await_confirmation(receipt, gas_strategy, required_confs) if rpc.is_active(): undo_thread = threading.Thread( @@ -663,6 +635,37 @@ def transfer( receipt._raise_if_reverted(exc) return receipt + def _await_confirmation( + self, receipt: TransactionReceipt, gas_strategy: Optional[GasABC], required_confs: int + ) -> TransactionReceipt: + # add to TxHistory before waiting for confirmation, this way the tx + # object is available if the user CTRL-C to stop waiting in the console + history._add_tx(receipt) + + if gas_strategy is not None: + gas_strategy._add_tx(receipt) + + if required_confs == 0: + return receipt + + receipt._confirmed.wait() + if receipt.status != -2: + return receipt + + # if transaction was dropped (status -2), find and return the tx that confirmed + replacements = history.filter( + sender=self, nonce=receipt.nonce, key=lambda k: k.status != -2 + ) + while True: + if not replacements: + raise TransactionError(f"Tx dropped without known replacement: {receipt.txid}") + if len(replacements) > 1: + # in case we have multiple tx's where the status is still unresolved + replacements = [i for i in replacements if i.status != 2] + time.sleep(0.5) + else: + return replacements[0] + class Account(_PrivateKeyAccount): From 3a6b5181950ebebdf205f026d2a983c1b392f260 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Wed, 18 Nov 2020 15:40:24 +0400 Subject: [PATCH 13/25] refactor: split `get_gas_price` and `update_gas_price` --- brownie/network/account.py | 17 ++++++++------ brownie/network/gas/bases.py | 37 +++++++++++++++++-------------- brownie/network/gas/strategies.py | 26 +++++++++++++--------- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/brownie/network/account.py b/brownie/network/account.py index 32f958f06..58f478a84 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -25,7 +25,7 @@ ) from brownie.utils import color -from .gas.bases import GasABC +from .gas.bases import GasABC, _add_to_gas_strategy_queue from .rpc import Rpc from .state import Chain, TxHistory, _revert_register from .transaction import TransactionReceipt @@ -375,13 +375,14 @@ def _gas_price(self, gas_price: Any = None) -> Tuple[Wei, Optional[GasABC]]: gas_price = CONFIG.active_network["settings"]["gas_price"] if isinstance(gas_price, GasABC): - if gas_price.get_gas_price.__code__.co_argcount: # type: ignore - return Wei(gas_price.get_gas_price(None, 0)), gas_price # type: ignore - else: - return Wei(gas_price.get_gas_price()), None # type: ignore + return Wei(gas_price.get_gas_price()), gas_price + + if isinstance(gas_price, Wei): + return gas_price, None if isinstance(gas_price, bool) or gas_price in (None, "auto"): return web3.eth.generateGasPrice(), None + return Wei(gas_price), None def _check_for_revert(self, tx: Dict) -> None: @@ -643,7 +644,7 @@ def _await_confirmation( history._add_tx(receipt) if gas_strategy is not None: - gas_strategy._add_tx(receipt) + _add_to_gas_strategy_queue(gas_strategy, receipt) if required_confs == 0: return receipt @@ -664,7 +665,9 @@ def _await_confirmation( replacements = [i for i in replacements if i.status != 2] time.sleep(0.5) else: - return replacements[0] + receipt = replacements[0] + receipt._await_confirmation(required_confs=required_confs) + return receipt class Account(_PrivateKeyAccount): diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index ddce5ea11..32cb3d393 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -8,30 +8,24 @@ class GasABC(ABC): - def _add_tx(self, txreceipt: Any) -> None: - if isinstance(self, SimpleGasStrategy): - return - - number = web3.eth.blockNumber if isinstance(self, BlockGasStrategy) else time.time() - _queue.append((self, txreceipt, number, number)) - _event.set() - - -class SimpleGasStrategy(GasABC): @abstractmethod def get_gas_price(self) -> int: raise NotImplementedError +class SimpleGasStrategy(GasABC): + pass + + class BlockGasStrategy(GasABC): block_duration = 2 - def __init__(self, block_duration: int = 2): + def __init__(self, block_duration: int = 2) -> None: self.block_duration = block_duration @abstractmethod - def get_gas_price(self, current_gas_price: int, elapsed_blocks: int) -> Optional[int]: + def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> Optional[int]: raise NotImplementedError @@ -43,11 +37,11 @@ def __init__(self, time_duration: int = 30) -> None: self.time_duration = time_duration @abstractmethod - def get_gas_price(self, current_gas_price: int, elapsed_time: int) -> Optional[int]: + def update_gas_price(self, last_gas_price: int, elapsed_time: int) -> Optional[int]: raise NotImplementedError -def _update_loop(): +def _update_loop() -> None: while True: if not _queue: _event.wait() @@ -64,7 +58,7 @@ def _update_loop(): if isinstance(gas_strategy, BlockGasStrategy): height = web3.eth.blockNumber if height - latest >= gas_strategy.block_duration: - gas_price = gas_strategy.get_gas_price(tx.gas_price, height - initial) + gas_price = gas_strategy.update_gas_price(tx.gas_price, height - initial) if gas_price is not None: try: tx = tx.replace(gas_price=gas_price, silent=True) @@ -72,9 +66,9 @@ def _update_loop(): except ValueError: pass - else: + elif isinstance(gas_strategy, TimeGasStrategy): if time.time() - latest >= gas_strategy.time_duration: - gas_price = gas_strategy.get_gas_price(tx.gas_price, time.time() - initial) + gas_price = gas_strategy.update_gas_price(tx.gas_price, time.time() - initial) if gas_price is not None: try: tx = tx.replace(gas_price=gas_price, silent=True) @@ -86,6 +80,15 @@ def _update_loop(): time.sleep(1) +def _add_to_gas_strategy_queue(gas_strategy: GasABC, txreceipt: Any) -> None: + if isinstance(gas_strategy, SimpleGasStrategy): + return + + number = web3.eth.blockNumber if isinstance(gas_strategy, BlockGasStrategy) else time.time() + _queue.append((gas_strategy, txreceipt, number, number)) + _event.set() + + _queue: deque = deque() _event = threading.Event() diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index d3d86e093..38b00a529 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -1,17 +1,20 @@ import threading import time +from typing import Dict, Optional import requests from .bases import BlockGasStrategy, SimpleGasStrategy -_gasnow = {"time": 0, "data": None} +_gasnow_update = 0 +_gasnow_data: Dict[str, int] = {} _gasnow_lock = threading.Lock() -def _fetch_gasnow(key): +def _fetch_gasnow(key: str) -> int: + global _gasnow_update with _gasnow_lock: - if time.time() - _gasnow["time"] > 15: + if time.time() - _gasnow_update > 15: data = None for i in range(12): response = requests.get( @@ -23,10 +26,10 @@ def _fetch_gasnow(key): data = response.json()["data"] if data is None: raise ValueError - _gasnow["time"] = data.pop("timestamp") // 1000 - _gasnow["data"] = data + _gasnow_update = data.pop("timestamp") // 1000 + _gasnow_data.update(data) - return _gasnow["data"][key] + return _gasnow_data[key] class GasNowStrategy(SimpleGasStrategy): @@ -35,7 +38,7 @@ def __init__(self, speed: str = "fast"): raise ValueError("`speed` must be one of: rapid, fast, standard, slow") self.speed = speed - def get_gas_price(self): + def get_gas_price(self) -> int: return _fetch_gasnow(self.speed) @@ -49,11 +52,12 @@ def __init__( self.speed = initial_speed self.increment = increment - def get_gas_price(self, current_gas_price, elapsed_blocks): - if current_gas_price is None: - return _fetch_gasnow(self.speed) + def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> Optional[int]: rapid_gas_price = _fetch_gasnow("rapid") - new_gas_price = max(int(current_gas_price * self.increment), _fetch_gasnow(self.speed)) + new_gas_price = max(int(last_gas_price * self.increment), _fetch_gasnow(self.speed)) if new_gas_price <= rapid_gas_price: return new_gas_price return None + + def get_gas_price(self) -> int: + return _fetch_gasnow(self.speed) From bb54e361f99d40f1721c793e89d1b33dfbfd1f67 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Wed, 18 Nov 2020 16:30:23 +0400 Subject: [PATCH 14/25] docs: add natspec for strategy classes and abcs --- brownie/network/gas/bases.py | 83 ++++++++++++++++++++++++++++++- brownie/network/gas/strategies.py | 26 ++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index 32cb3d393..28d7ff008 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -8,16 +8,48 @@ class GasABC(ABC): + """ + Base ABC for all gas strategies. + + This class should not be directly subclassed from. Instead, use + `SimpleGasStrategy`, `BlockGasStrategy` or `TimeGasStrategy`. + """ + @abstractmethod def get_gas_price(self) -> int: + """ + Return the initial gas price for a transaction. + + Returns + ------- + int + Gas price, given as an integer in wei. + """ raise NotImplementedError class SimpleGasStrategy(GasABC): - pass + """ + Abstract base class for simple gas strategies. + + Simple gas strategies are called once to provide a gas price + at the time a transaction is broadcasted. Transactions using simple + gas strategies are not automatically rebroadcasted. + + Subclass from this ABC to implement your own simple gas strategy. + """ class BlockGasStrategy(GasABC): + """ + Abstract base class for block gas strategies. + + Block gas strategies are called every `block_duration` blocks and + can be used to automatically rebroadcast a pending transaction with + a higher gas price. + + Subclass from this ABC to implement your own block gas strategy. + """ block_duration = 2 @@ -26,10 +58,39 @@ def __init__(self, block_duration: int = 2) -> None: @abstractmethod def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> Optional[int]: + """ + Return an updated gas price. + + This method is called every `block_duration` blocks while a transaction + is still pending. If the return value is an integer, the transaction + is rebroadcasted with the new gas price. + + Arguments + --------- + last_gas_price : int + The gas price of the most recently broadcasted transaction. + elapsed_blocks : int + The total number of blocks that have been mined since the first + transaction using this strategy was broadcasted. + + Returns + ------- + int, optional + New gas price to rebroadcast the transaction with. + """ raise NotImplementedError class TimeGasStrategy(GasABC): + """ + Abstract base class for time gas strategies. + + Time gas strategies are called every `time_duration` seconds and + can be used to automatically rebroadcast a pending transaction with + a higher gas price. + + Subclass from this ABC to implement your own time gas strategy. + """ time_duration = 30 @@ -38,6 +99,26 @@ def __init__(self, time_duration: int = 30) -> None: @abstractmethod def update_gas_price(self, last_gas_price: int, elapsed_time: int) -> Optional[int]: + """ + Return an updated gas price. + + This method is called every `time_duration` seconds while a transaction + is still pending. If the return value is an integer, the transaction + is rebroadcasted with the new gas price. + + Arguments + --------- + last_gas_price : int + The gas price of the most recently broadcasted transaction. + elapsed_time : int + The number of seconds that have passed since the first + transaction using this strategy was broadcasted. + + Returns + ------- + int, optional + New gas price to rebroadcast the transaction with. + """ raise NotImplementedError diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index 38b00a529..58e638855 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -33,6 +33,22 @@ def _fetch_gasnow(key: str) -> int: class GasNowStrategy(SimpleGasStrategy): + """ + Gas strategy for determing a price using the GasNow API. + + GasNow returns 4 possible prices: + + rapid: the median gas prices for all transactions currently included + in the mining block + fast: the gas price transaction "N", the minimum priced tx currently + included in the mining block + standard: the gas price of the Max(2N, 500th) transaction in the mempool + slow: the gas price of the max(5N, 1000th) transaction within the mempool + + Visit https://www.gasnow.org/ for more information on how GasNow + calculates gas prices. + """ + def __init__(self, speed: str = "fast"): if speed not in ("rapid", "fast", "standard", "slow"): raise ValueError("`speed` must be one of: rapid, fast, standard, slow") @@ -43,6 +59,16 @@ def get_gas_price(self) -> int: class GasNowScalingStrategy(BlockGasStrategy): + """ + Block based scaling gas strategy using the GasNow API. + + The initial gas price is set according to `initial_speed`. The gas price + for each subsequent transaction is increased by multiplying the previous gas + price by `increment`, or increasing to the current `initial_speed` gas price, + whichever is higher. No repricing occurs if the new gas price would exceed + the current "rapid" price as given by the API. + """ + def __init__( self, initial_speed: str = "standard", increment: float = 1.1, block_duration: int = 2 ): From 61ce105dd346b889bd2eccb95f1298f3a9c826ad Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Wed, 18 Nov 2020 18:26:51 +0400 Subject: [PATCH 15/25] fix: improve console output for blocking replaced tx's --- brownie/network/gas/bases.py | 4 ++-- brownie/network/transaction.py | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index 28d7ff008..66be3aca9 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -142,7 +142,7 @@ def _update_loop() -> None: gas_price = gas_strategy.update_gas_price(tx.gas_price, height - initial) if gas_price is not None: try: - tx = tx.replace(gas_price=gas_price, silent=True) + tx = tx.replace(gas_price=gas_price) latest = web3.eth.blockNumber except ValueError: pass @@ -152,7 +152,7 @@ def _update_loop() -> None: gas_price = gas_strategy.update_gas_price(tx.gas_price, time.time() - initial) if gas_price is not None: try: - tx = tx.replace(gas_price=gas_price, silent=True) + tx = tx.replace(gas_price=gas_price) latest = time.time() except ValueError: pass diff --git a/brownie/network/transaction.py b/brownie/network/transaction.py index f0703ed0f..1071ab90d 100644 --- a/brownie/network/transaction.py +++ b/brownie/network/transaction.py @@ -148,7 +148,7 @@ def __init__( if isinstance(txid, bytes): txid = HexBytes(txid).hex() if not self._silent: - print(f"Transaction sent: {color('bright blue')}{txid}{color}") + print(f"\rTransaction sent: {color('bright blue')}{txid}{color}") # this event is set once the transaction is confirmed or dropped # it is used to waiting during blocking transaction actions @@ -334,7 +334,7 @@ def wait(self, required_confs: int) -> None: return time.sleep(1) - self._await_confirmation(tx, required_confs) + self._await_confirmation(tx["blockNumber"], required_confs) def _raise_if_reverted(self, exc: Any) -> None: if self.status or CONFIG.mode == "console": @@ -384,22 +384,22 @@ def _await_transaction(self, required_confs: int, is_blocking: bool) -> None: # await confirmation of tx in a separate thread which is blocking if # required_confs > 0 or tx has already confirmed (`blockNumber` != None) confirm_thread = threading.Thread( - target=self._await_confirmation, args=(tx, required_confs), daemon=True + target=self._await_confirmation, args=(tx["blockNumber"], required_confs), daemon=True ) confirm_thread.start() if is_blocking and (required_confs > 0 or tx["blockNumber"]): confirm_thread.join() - def _await_confirmation(self, tx: Dict, required_confs: int = 1) -> None: - if not tx["blockNumber"] and not self._silent and required_confs > 0: + def _await_confirmation(self, block_number: int = None, required_confs: int = 1) -> None: + block_number = block_number or self.block_number + if not block_number and not self._silent and required_confs > 0: if required_confs == 1: - print("Waiting for confirmation...") + sys.stdout.write("\rWaiting for confirmation... ") else: sys.stdout.write( - f"\rRequired confirmations: {color('bright yellow')}0/" - f"{required_confs}{color}" + f"\rRequired confirmations: {color('bright yellow')}0/{required_confs}{color}" ) - sys.stdout.flush() + sys.stdout.flush() # await first confirmation while True: @@ -432,7 +432,7 @@ def _await_confirmation(self, tx: Dict, required_confs: int = 1) -> None: # check if tx is still in mempool, this will raise otherwise tx = web3.eth.getTransaction(self.txid) self.block_number = None - return self._await_confirmation(tx, required_confs) + return self._await_confirmation(tx["blockNumber"], required_confs) if required_confs - self.confirmations != remaining_confs: remaining_confs = required_confs - self.confirmations if not self._silent: @@ -510,7 +510,7 @@ def _confirm_output(self) -> str: revert_msg = self.revert_msg if web3.supports_traces else None status = f"({color('bright red')}{revert_msg or 'reverted'}{color}) " result = ( - f" {self._full_name()} confirmed {status}- " + f"\r {self._full_name()} confirmed {status}- " f"Block: {color('bright blue')}{self.block_number}{color} " f"Gas used: {color('bright blue')}{self.gas_used}{color} " f"({color('bright blue')}{self.gas_used / self.gas_limit:.2%}{color})" From 18dc3113139c623fc4f8abd5e29322bc70ad2719 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Wed, 18 Nov 2020 19:38:34 +0400 Subject: [PATCH 16/25] feat: silence pending tx output on keyboard interrupt --- brownie/network/account.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/brownie/network/account.py b/brownie/network/account.py index 58f478a84..080cb13f4 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -640,7 +640,7 @@ def _await_confirmation( self, receipt: TransactionReceipt, gas_strategy: Optional[GasABC], required_confs: int ) -> TransactionReceipt: # add to TxHistory before waiting for confirmation, this way the tx - # object is available if the user CTRL-C to stop waiting in the console + # object is available if the user exits blocking via keyboard interrupt history._add_tx(receipt) if gas_strategy is not None: @@ -649,7 +649,17 @@ def _await_confirmation( if required_confs == 0: return receipt - receipt._confirmed.wait() + try: + receipt._confirmed.wait() + except KeyboardInterrupt: + # set related transactions as silent + receipt._silent = True + for receipt in history.filter( + sender=self, nonce=receipt.nonce, key=lambda k: k.status != -2 + ): + receipt._silent = True + raise + if receipt.status != -2: return receipt From 9e046044464de3c6ea931500e9aaf902b3a36377 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Thu, 19 Nov 2020 00:37:22 +0400 Subject: [PATCH 17/25] fix: updates to gasnow strategies --- brownie/network/gas/strategies.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index 58e638855..2e827c525 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -66,24 +66,29 @@ class GasNowScalingStrategy(BlockGasStrategy): for each subsequent transaction is increased by multiplying the previous gas price by `increment`, or increasing to the current `initial_speed` gas price, whichever is higher. No repricing occurs if the new gas price would exceed - the current "rapid" price as given by the API. + the current `max_speed` price as given by the API. """ def __init__( - self, initial_speed: str = "standard", increment: float = 1.1, block_duration: int = 2 + self, + initial_speed: str = "standard", + max_speed: str = "rapid", + increment: float = 1.125, + block_duration: int = 2, ): super().__init__(block_duration) if initial_speed not in ("rapid", "fast", "standard", "slow"): raise ValueError("`initial_speed` must be one of: rapid, fast, standard, slow") - self.speed = initial_speed + self.initial_speed = initial_speed + self.max_speed = max_speed self.increment = increment def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> Optional[int]: - rapid_gas_price = _fetch_gasnow("rapid") - new_gas_price = max(int(last_gas_price * self.increment), _fetch_gasnow(self.speed)) - if new_gas_price <= rapid_gas_price: + max_gas_price = _fetch_gasnow(self.max_speed) + new_gas_price = max(int(last_gas_price * self.increment), _fetch_gasnow(self.initial_speed)) + if new_gas_price <= max_gas_price: return new_gas_price return None def get_gas_price(self) -> int: - return _fetch_gasnow(self.speed) + return _fetch_gasnow(self.initial_speed) From 617699b7499a879cab22689ff8147cef23374eb0 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Thu, 19 Nov 2020 16:13:02 +0400 Subject: [PATCH 18/25] style: always return int from gas strategy --- brownie/network/gas/bases.py | 18 +++++++++--------- brownie/network/gas/strategies.py | 13 +++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index 66be3aca9..40213d1cf 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -2,7 +2,7 @@ import time from abc import ABC, abstractmethod from collections import deque -from typing import Any, Optional +from typing import Any from brownie.network.web3 import web3 @@ -57,13 +57,13 @@ def __init__(self, block_duration: int = 2) -> None: self.block_duration = block_duration @abstractmethod - def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> Optional[int]: + def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> int: """ Return an updated gas price. This method is called every `block_duration` blocks while a transaction - is still pending. If the return value is an integer, the transaction - is rebroadcasted with the new gas price. + is still pending. If the return value is at least 10% higher than the + current gas price, the transaction is rebroadcasted with the new gas price. Arguments --------- @@ -98,13 +98,13 @@ def __init__(self, time_duration: int = 30) -> None: self.time_duration = time_duration @abstractmethod - def update_gas_price(self, last_gas_price: int, elapsed_time: int) -> Optional[int]: + def update_gas_price(self, last_gas_price: int, elapsed_time: int) -> int: """ Return an updated gas price. This method is called every `time_duration` seconds while a transaction - is still pending. If the return value is an integer, the transaction - is rebroadcasted with the new gas price. + is still pending. If the return value is at least 10% higher than the + current gas price, the transaction is rebroadcasted with the new gas price. Arguments --------- @@ -140,7 +140,7 @@ def _update_loop() -> None: height = web3.eth.blockNumber if height - latest >= gas_strategy.block_duration: gas_price = gas_strategy.update_gas_price(tx.gas_price, height - initial) - if gas_price is not None: + if gas_price >= int(tx.gas_price * 1.1): try: tx = tx.replace(gas_price=gas_price) latest = web3.eth.blockNumber @@ -150,7 +150,7 @@ def _update_loop() -> None: elif isinstance(gas_strategy, TimeGasStrategy): if time.time() - latest >= gas_strategy.time_duration: gas_price = gas_strategy.update_gas_price(tx.gas_price, time.time() - initial) - if gas_price is not None: + if gas_price >= int(tx.gas_price * 1.1): try: tx = tx.replace(gas_price=gas_price) latest = time.time() diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index 2e827c525..b962da97f 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -1,6 +1,6 @@ import threading import time -from typing import Dict, Optional +from typing import Dict import requests @@ -83,12 +83,13 @@ def __init__( self.max_speed = max_speed self.increment = increment - def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> Optional[int]: + def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> int: + initial_gas_price = _fetch_gasnow(self.initial_speed) max_gas_price = _fetch_gasnow(self.max_speed) - new_gas_price = max(int(last_gas_price * self.increment), _fetch_gasnow(self.initial_speed)) - if new_gas_price <= max_gas_price: - return new_gas_price - return None + + incremented_gas_price = int(last_gas_price * self.increment) + new_gas_price = max(initial_gas_price, incremented_gas_price) + return min(max_gas_price, new_gas_price) def get_gas_price(self) -> int: return _fetch_gasnow(self.initial_speed) From 196fd7e5d3f7d001e98871fdc178afd32a027ddb Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Thu, 19 Nov 2020 22:47:24 +0400 Subject: [PATCH 19/25] refactor: use generator functions for scaling strategies, remove strategy queue --- brownie/network/account.py | 41 +++++-- brownie/network/gas/bases.py | 189 ++++++++++++++---------------- brownie/network/gas/strategies.py | 28 +++-- 3 files changed, 133 insertions(+), 125 deletions(-) diff --git a/brownie/network/account.py b/brownie/network/account.py index 080cb13f4..2c652f655 100644 --- a/brownie/network/account.py +++ b/brownie/network/account.py @@ -3,9 +3,10 @@ import json import threading import time +from collections.abc import Iterator from getpass import getpass from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import eth_account import eth_keys @@ -25,7 +26,7 @@ ) from brownie.utils import color -from .gas.bases import GasABC, _add_to_gas_strategy_queue +from .gas.bases import GasABC from .rpc import Rpc from .state import Chain, TxHistory, _revert_register from .transaction import TransactionReceipt @@ -370,20 +371,29 @@ def _gas_limit( return Wei(gas_limit) - def _gas_price(self, gas_price: Any = None) -> Tuple[Wei, Optional[GasABC]]: + def _gas_price(self, gas_price: Any = None) -> Tuple[Wei, Optional[GasABC], Optional[Iterator]]: + # returns the gas price, gas strategy object, and active gas strategy iterator if gas_price is None: gas_price = CONFIG.active_network["settings"]["gas_price"] if isinstance(gas_price, GasABC): - return Wei(gas_price.get_gas_price()), gas_price + value = gas_price.get_gas_price() + if isinstance(value, Iterator): + # if `get_gas_price` returns an interator, this is a gas strategy + # intended for rebroadcasting. we need to retain both the strategy + # object and the active gas price iterator + return Wei(next(value)), gas_price, value + else: + # for simple strategies, we can simply use the generated gas price + return Wei(value), None, None if isinstance(gas_price, Wei): - return gas_price, None + return gas_price, None, None if isinstance(gas_price, bool) or gas_price in (None, "auto"): - return web3.eth.generateGasPrice(), None + return web3.eth.generateGasPrice(), None, None - return Wei(gas_price), None + return Wei(gas_price), None, None def _check_for_revert(self, tx: Dict) -> None: try: @@ -439,7 +449,7 @@ def deploy( silent = bool(CONFIG.mode == "test" or CONFIG.argv["silent"]) with self._lock: try: - gas_price, gas_strategy = self._gas_price(gas_price) + gas_price, gas_strategy, gas_iter = self._gas_price(gas_price) gas_limit = Wei(gas_limit) or self._gas_limit( None, amount, gas_price, gas_buffer, data ) @@ -472,7 +482,7 @@ def deploy( revert_data=revert_data, ) - receipt = self._await_confirmation(receipt, gas_strategy, required_confs) + receipt = self._await_confirmation(receipt, required_confs, gas_strategy, gas_iter) add_thread = threading.Thread(target=contract._add_from_tx, args=(receipt,), daemon=True) add_thread.start() @@ -587,7 +597,7 @@ def transfer( if silent is None: silent = bool(CONFIG.mode == "test" or CONFIG.argv["silent"]) with self._lock: - gas_price, gas_strategy = self._gas_price(gas_price) + gas_price, gas_strategy, gas_iter = self._gas_price(gas_price) gas_limit = Wei(gas_limit) or self._gas_limit(to, amount, gas_price, gas_buffer, data) tx = { "from": self.address, @@ -618,7 +628,7 @@ def transfer( revert_data=revert_data, ) - receipt = self._await_confirmation(receipt, gas_strategy, required_confs) + receipt = self._await_confirmation(receipt, required_confs, gas_strategy, gas_iter) if rpc.is_active(): undo_thread = threading.Thread( @@ -637,16 +647,21 @@ def transfer( return receipt def _await_confirmation( - self, receipt: TransactionReceipt, gas_strategy: Optional[GasABC], required_confs: int + self, + receipt: TransactionReceipt, + required_confs: int, + gas_strategy: Optional[GasABC], + gas_iter: Optional[Iterator], ) -> TransactionReceipt: # add to TxHistory before waiting for confirmation, this way the tx # object is available if the user exits blocking via keyboard interrupt history._add_tx(receipt) if gas_strategy is not None: - _add_to_gas_strategy_queue(gas_strategy, receipt) + gas_strategy.run(receipt, gas_iter) # type: ignore if required_confs == 0: + receipt._silent = True return receipt try: diff --git a/brownie/network/gas/bases.py b/brownie/network/gas/bases.py index 40213d1cf..e83a57ca4 100644 --- a/brownie/network/gas/bases.py +++ b/brownie/network/gas/bases.py @@ -1,8 +1,8 @@ +import inspect import threading import time from abc import ABC, abstractmethod -from collections import deque -from typing import Any +from typing import Any, Generator, Iterator, Union from brownie.network.web3 import web3 @@ -16,17 +16,65 @@ class GasABC(ABC): """ @abstractmethod - def get_gas_price(self) -> int: + def get_gas_price(self) -> Union[Generator[int, None, None], int]: + raise NotImplementedError + + +class ScalingGasABC(GasABC): + """ + Base ABC for scaling gas strategies. + + This class should not be directly subclassed from. + Instead, use `BlockGasStrategy` or `TimeGasStrategy`. + """ + + duration: int + + def __new__(cls, *args: Any, **kwargs: Any) -> object: + obj = super().__new__(cls) + if not inspect.isgeneratorfunction(cls.get_gas_price): + raise TypeError("Scaling strategy must implement get_gas_price as a generator function") + return obj + + @abstractmethod + def interval(self) -> int: """ - Return the initial gas price for a transaction. + Return "now" as it relates to the scaling strategy. - Returns - ------- - int - Gas price, given as an integer in wei. + This can be e.g. the current time or block height. It is used in combination + with `duration` to determine when to rebroadcast a transaction. """ raise NotImplementedError + def _loop(self, receipt: Any, gas_iter: Iterator) -> None: + while web3.eth.getTransactionCount(str(receipt.sender)) < receipt.nonce: + # do not run scaling strategy while prior tx's are still pending + time.sleep(5) + + latest_interval = self.interval() + while True: + if web3.eth.getTransactionCount(str(receipt.sender)) > receipt.nonce: + break + + if self.interval() - latest_interval >= self.duration: + gas_price = next(gas_iter) + if gas_price >= int(receipt.gas_price * 1.1): + try: + receipt = receipt.replace(gas_price=gas_price) + latest_interval = self.interval() + except ValueError: + pass + time.sleep(2) + + def run(self, receipt: Any, gas_iter: Iterator) -> None: + thread = threading.Thread( + target=self._loop, + args=(receipt, gas_iter), + daemon=True, + name=f"Gas strategy {receipt.txid}", + ) + thread.start() + class SimpleGasStrategy(GasABC): """ @@ -39,8 +87,20 @@ class SimpleGasStrategy(GasABC): Subclass from this ABC to implement your own simple gas strategy. """ + @abstractmethod + def get_gas_price(self) -> int: + """ + Return the initial gas price for a transaction. + + Returns + ------- + int + Gas price, given as an integer in wei. + """ + raise NotImplementedError + -class BlockGasStrategy(GasABC): +class BlockGasStrategy(ScalingGasABC): """ Abstract base class for block gas strategies. @@ -51,37 +111,28 @@ class BlockGasStrategy(GasABC): Subclass from this ABC to implement your own block gas strategy. """ - block_duration = 2 + duration = 2 def __init__(self, block_duration: int = 2) -> None: - self.block_duration = block_duration + self.duration = block_duration + + def interval(self) -> int: + return web3.eth.blockNumber @abstractmethod - def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> int: + def get_gas_price(self) -> Generator[int, None, None]: """ - Return an updated gas price. - - This method is called every `block_duration` blocks while a transaction - is still pending. If the return value is at least 10% higher than the - current gas price, the transaction is rebroadcasted with the new gas price. - - Arguments - --------- - last_gas_price : int - The gas price of the most recently broadcasted transaction. - elapsed_blocks : int - The total number of blocks that have been mined since the first - transaction using this strategy was broadcasted. + Generator function to yield gas prices for a transaction. Returns ------- - int, optional - New gas price to rebroadcast the transaction with. + int + Gas price, given as an integer in wei. """ raise NotImplementedError -class TimeGasStrategy(GasABC): +class TimeGasStrategy(ScalingGasABC): """ Abstract base class for time gas strategies. @@ -92,86 +143,22 @@ class TimeGasStrategy(GasABC): Subclass from this ABC to implement your own time gas strategy. """ - time_duration = 30 + duration = 30 def __init__(self, time_duration: int = 30) -> None: - self.time_duration = time_duration + self.duration = time_duration + + def interval(self) -> int: + return int(time.time()) @abstractmethod - def update_gas_price(self, last_gas_price: int, elapsed_time: int) -> int: + def get_gas_price(self) -> Generator[int, None, None]: """ - Return an updated gas price. - - This method is called every `time_duration` seconds while a transaction - is still pending. If the return value is at least 10% higher than the - current gas price, the transaction is rebroadcasted with the new gas price. - - Arguments - --------- - last_gas_price : int - The gas price of the most recently broadcasted transaction. - elapsed_time : int - The number of seconds that have passed since the first - transaction using this strategy was broadcasted. + Generator function to yield gas prices for a transaction. Returns ------- - int, optional - New gas price to rebroadcast the transaction with. + int + Gas price, given as an integer in wei. """ raise NotImplementedError - - -def _update_loop() -> None: - while True: - if not _queue: - _event.wait() - _event.clear() - - try: - gas_strategy, tx, initial, latest = _queue.popleft() - except IndexError: - continue - - if tx.status >= 0: - continue - - if isinstance(gas_strategy, BlockGasStrategy): - height = web3.eth.blockNumber - if height - latest >= gas_strategy.block_duration: - gas_price = gas_strategy.update_gas_price(tx.gas_price, height - initial) - if gas_price >= int(tx.gas_price * 1.1): - try: - tx = tx.replace(gas_price=gas_price) - latest = web3.eth.blockNumber - except ValueError: - pass - - elif isinstance(gas_strategy, TimeGasStrategy): - if time.time() - latest >= gas_strategy.time_duration: - gas_price = gas_strategy.update_gas_price(tx.gas_price, time.time() - initial) - if gas_price >= int(tx.gas_price * 1.1): - try: - tx = tx.replace(gas_price=gas_price) - latest = time.time() - except ValueError: - pass - - _queue.append((gas_strategy, tx, initial, latest)) - time.sleep(1) - - -def _add_to_gas_strategy_queue(gas_strategy: GasABC, txreceipt: Any) -> None: - if isinstance(gas_strategy, SimpleGasStrategy): - return - - number = web3.eth.blockNumber if isinstance(gas_strategy, BlockGasStrategy) else time.time() - _queue.append((gas_strategy, txreceipt, number, number)) - _event.set() - - -_queue: deque = deque() -_event = threading.Event() - -_repricing_thread = threading.Thread(target=_update_loop, daemon=True) -_repricing_thread.start() diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index b962da97f..6d9d201f3 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -1,6 +1,6 @@ import threading import time -from typing import Dict +from typing import Dict, Generator import requests @@ -24,6 +24,7 @@ def _fetch_gasnow(key: str) -> int: time.sleep(5) continue data = response.json()["data"] + break if data is None: raise ValueError _gasnow_update = data.pop("timestamp") // 1000 @@ -83,13 +84,18 @@ def __init__( self.max_speed = max_speed self.increment = increment - def update_gas_price(self, last_gas_price: int, elapsed_blocks: int) -> int: - initial_gas_price = _fetch_gasnow(self.initial_speed) - max_gas_price = _fetch_gasnow(self.max_speed) - - incremented_gas_price = int(last_gas_price * self.increment) - new_gas_price = max(initial_gas_price, incremented_gas_price) - return min(max_gas_price, new_gas_price) - - def get_gas_price(self) -> int: - return _fetch_gasnow(self.initial_speed) + def get_gas_price(self) -> Generator[int, None, None]: + last_gas_price = _fetch_gasnow(self.initial_speed) + yield last_gas_price + + while True: + # increment the last price by `increment` or use the new + # `initial_speed` value, whichever is higher + initial_gas_price = _fetch_gasnow(self.initial_speed) + incremented_gas_price = int(last_gas_price * self.increment) + new_gas_price = max(initial_gas_price, incremented_gas_price) + + # do not exceed the current `max_speed` price + max_gas_price = _fetch_gasnow(self.max_speed) + last_gas_price = min(max_gas_price, new_gas_price) + yield last_gas_price From a335cf0cb7b2ff97baa8f2d9438209b77a9df22b Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 20 Nov 2020 03:01:55 +0400 Subject: [PATCH 20/25] feat: linear and exponential scaling strategies --- brownie/network/gas/strategies.py | 76 ++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index 6d9d201f3..493675460 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -1,10 +1,11 @@ +import itertools import threading import time from typing import Dict, Generator import requests -from .bases import BlockGasStrategy, SimpleGasStrategy +from .bases import BlockGasStrategy, SimpleGasStrategy, TimeGasStrategy _gasnow_update = 0 _gasnow_data: Dict[str, int] = {} @@ -33,6 +34,79 @@ def _fetch_gasnow(key: str) -> int: return _gasnow_data[key] +class LinearScalingStrategy(TimeGasStrategy): + """ + Gas strategy for linear gas price increase. + + Arguments + --------- + initial_gas_price : int + The initial gas price to use in the first transaction + max_gas_price : int + The maximum gas price to use + increment : float + Multiplier applied to the previous gas price in order to determine the new gas price + time_duration : int + Number of seconds between transactions + """ + + def __init__( + self, + initial_gas_price: int, + max_gas_price: int, + increment: float = 1.125, + time_duration: int = 30, + ): + super().__init__(time_duration) + self.initial_gas_price = initial_gas_price + self.max_gas_price = max_gas_price + self.increment = increment + + def get_gas_price(self) -> Generator[int, None, None]: + last_gas_price = self.initial_gas_price + yield last_gas_price + + while True: + last_gas_price = min(int(last_gas_price * self.increment), self.max_gas_price) + yield last_gas_price + + +class ExponentialScalingStrategy(TimeGasStrategy): + """ + Gas strategy for exponential increasing gas prices. + + The gas price for each subsequent transaction is calculated as the previous price + multiplied by `1.1 ** n` where n is the number of transactions that have been broadcast. + In this way the price increase starts gradually and ramps up until confirmation. + + Arguments + --------- + initial_gas_price : int + The initial gas price to use in the first transaction + max_gas_price : int + The maximum gas price to use + increment : float + Multiplier applied to the previous gas price in order to determine the new gas price + time_duration : int + Number of seconds between transactions + """ + + def __init__( + self, initial_gas_price: int, max_gas_price: int, time_duration: int = 30, + ): + super().__init__(time_duration) + self.initial_gas_price = initial_gas_price + self.max_gas_price = max_gas_price + + def get_gas_price(self) -> Generator[int, None, None]: + last_gas_price = self.initial_gas_price + yield last_gas_price + + for i in itertools.count(1): + last_gas_price = int(last_gas_price * 1.1 ** i) + yield min(last_gas_price, self.max_gas_price) + + class GasNowStrategy(SimpleGasStrategy): """ Gas strategy for determing a price using the GasNow API. From 36db8f4cdab40448092e1dd3a3e924d3b566b074 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 20 Nov 2020 22:36:42 +0400 Subject: [PATCH 21/25] feat: graphql gas strategy --- brownie/network/gas/strategies.py | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index 493675460..54824e2bf 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -5,6 +5,9 @@ import requests +from brownie.exceptions import RPCRequestError +from brownie.network.web3 import web3 + from .bases import BlockGasStrategy, SimpleGasStrategy, TimeGasStrategy _gasnow_update = 0 @@ -173,3 +176,37 @@ def get_gas_price(self) -> Generator[int, None, None]: max_gas_price = _fetch_gasnow(self.max_speed) last_gas_price = min(max_gas_price, new_gas_price) yield last_gas_price + + +class GethMempoolStrategy(BlockGasStrategy): + """ + Block based scaling gas strategy using the GraphQL and the Geth mempool. + + The yielded gas price is determined by sorting transactions in the mempool + according to gas price, and returning the price of the transaction at `position`. + This is the same technique used by the GasNow API. + + A position of 500 should place a transaction within the 2nd block to be mined. + A position of 200 or less should place it within the next block. + """ + + def __init__(self, position: int = 500, graphql_endpoint: str = None, block_duration: int = 2): + super().__init__(block_duration) + self.position = position + if graphql_endpoint is None: + graphql_endpoint = f"{web3.provider.endpoint_uri}/graphql" # type: ignore + self.graphql_endpoint = graphql_endpoint + + def get_gas_price(self) -> Generator[int, None, None]: + query = "{ pending { transactions { gasPrice }}}" + + while True: + response = requests.post(self.graphql_endpoint, json={"query": query}) + response.raise_for_status() + if "error" in response.json(): + raise RPCRequestError("could not fetch mempool, run geth with `--graphql` flag") + + data = response.json()["data"]["pending"]["transactions"] + + prices = sorted((int(x["gasPrice"], 16) for x in data), reverse=True) + yield prices[: self.position][-1] From b7a36bc7edb3805d7508c56ca6f8b6b664dec8aa Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Thu, 19 Nov 2020 05:36:13 +0400 Subject: [PATCH 22/25] docs: add documentation for gas strategies --- docs/api-network.rst | 140 +++++++++++++++++++++++++++++++++++++++++ docs/core-accounts.rst | 2 + docs/core-gas.rst | 131 ++++++++++++++++++++++++++++++++++++++ docs/toctree.rst | 1 + 4 files changed, 274 insertions(+) create mode 100644 docs/core-gas.rst diff --git a/docs/api-network.rst b/docs/api-network.rst index f9d97d37f..56107e050 100644 --- a/docs/api-network.rst +++ b/docs/api-network.rst @@ -1461,6 +1461,146 @@ Internal Methods >>> e {} + +``brownie.network.gas`` +======================= + +The ``gas`` module contains gas strategy classes, as well as abstract base classes for buiding your own gas strategies. + +Gas Strategies +-------------- + +.. py:class:: brownie.network.gas.strategies.ExponentialScalingStrategy(initial_gas_price, max_gas_price, time_duration=30) + + Time based scaling strategy for exponential gas price increase. + + The gas price for each subsequent transaction is calculated as the previous price multiplied by `1.1 ** n` where n is the number of transactions that have been broadcast. In this way the price increase starts gradually and ramps up until confirmation. + + * ``initial_gas_price``: The initial gas price to use in the first transaction + * ``max_gas_price``: The maximum gas price to use + * ``time_duration``: Number of seconds between transactions + + .. code-block:: python + + >>> from brownie.network.gas.strategies import ExponentialScalingStrategy + >>> gas_strategy = ExponentialScalingStrategy("10 gwei", "50 gwei") + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.GasNowStrategy(speed="fast") + + Gas strategy for determing a price using the `GasNow `_ API. + + * ``speed``: The gas price to use based on the API call. Options are rapid, fast, standard and slow. + + .. code-block:: python + + >>> from brownie.network.gas.strategies import GasNowStrategy + >>> gas_strategy = GasNowStrategy("fast") + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.GasNowScalingStrategy(initial_speed="standard", max_speed="rapid", increment=1.125, block_duration=2) + + Block based scaling gas strategy using the GasNow API. + + * ``initial_speed``: The initial gas price to use when broadcasting the first transaction. Options are rapid, fast, standard and slow. + * ``max_speed``: The maximum gas price to use when replacing the transaction. Options are rapid, fast, standard and slow. + * ``increment``: A multiplier applied to the most recently used gas price in order to determine the new gas price. If the incremented value is less than or equal to the current ``max_speed`` rate, a new transaction is broadcasted. If the current rate for ``initial_speed`` is greater than the incremented rate, it is used instead. + * ``block_duration``: The number of blocks to wait between broadcasting new transactions. + + .. code-block:: python + + >>> from brownie.network.gas.strategies import GasNowScalingStrategy + >>> gas_strategy = GasNowScalingStrategy("standard", increment=1.125, block_duration=2) + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.GethMempoolStrategy(position=500, graphql_endpoint=None, block_duration=2) + + Block based scaling gas strategy using Geth's `GraphQL interface `_. + + In order to use this strategy you must be connecting via a Geth node with GraphQL enabled. + + The yielded gas price is determined by sorting transactions in the mempool according to gas price, and returning the price of the transaction at `position`. This is the same technique used by the GasNow API. + + * A position of 200 or less usually places a transaction within the mining block. + * A position of 500 usually places a transaction within the 2nd pending block. + + .. code-block:: python + + >>> from brownie.network.gas.strategies import GethMempoolStrategy + >>> gas_strategy = GethMempoolStrategy(200) + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.LinearScalingStrategy(initial_gas_price, max_gas_price, increment=1.125, time_duration=30) + + Time based scaling strategy for linear gas price increase. + + * ``initial_gas_price``: The initial gas price to use in the first transaction + * ``max_gas_price``: The maximum gas price to use + * ``increment``: Multiplier applied to the previous gas price in order to determine the new gas price + * ``time_duration``: Number of seconds between transactions + + .. code-block:: python + + >>> from brownie.network.gas.strategies import LinearScalingStrategy + >>> gas_strategy = LinearScalingStrategy("10 gwei", "50 gwei", 1.1) + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. _api-network-gas-abc: + +Gas Strategy ABCs +----------------- + +`Abstract base classes `_ for building your own gas strategies. + +Simple Strategies +***************** + +.. py:class:: brownie.network.gas.bases.SimpleGasStrategy + + Abstract base class for simple gas strategies. + + Simple gas strategies are called once to provide a dynamically genreated gas price at the time a transaction is broadcasted. Transactions using simple gas strategies are not automatically rebroadcasted. + +Simple Strategy Abstract Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To implement a simple gas strategy, subclass :func:`SimpleGasStrategy ` and include the following method: + +.. py:method:: SimpleGasStrategy.get_gas_price(self) -> int: + + Return the gas price for a transaction. + +Scaling Strategies +****************** + +.. py:class:: brownie.network.gas.bases.BlockGasStrategy(duration=2) + + Abstract base class for block-based gas strategies. + + Block gas strategies are called every ``duration`` blocks and can be used to automatically rebroadcast a pending transaction with a higher gas price. + +.. py:class:: brownie.network.gas.bases.TimeGasStrategy(duration=30) + + Abstract base class for time-based gas strategies. + + Time gas strategies are called every ``duration`` seconds and can be used to automatically rebroadcast a pending transaction with a higher gas price. + +Scaling Strategy Abstract Methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To implement a scaling strategy, subclass one of the above ABCs and implement the following generator function: + +.. py:method:: BlockGasStrategy.get_gas_price(self) -> Generator[int]: + + Generator function that yields a new gas price each time it is called. + + The produced generator is called every ``duration`` seconds while a transaction is still pending. Each call must yield a new gas price as an integer. If the newly yielded value is at least 10% higher than the current gas price, the transaction is rebroadcasted with the new gas price. + ``brownie.network.state`` ========================= diff --git a/docs/core-accounts.rst b/docs/core-accounts.rst index f8eb03bca..e9d998494 100644 --- a/docs/core-accounts.rst +++ b/docs/core-accounts.rst @@ -70,6 +70,8 @@ In a development environment, it is possible to send transactions from an addres See :ref:`local-accounts` for more information on working with accounts. +.. _core-accounts-non-blocking: + Broadcasting Multiple Transactions ================================== diff --git a/docs/core-gas.rst b/docs/core-gas.rst new file mode 100644 index 000000000..8764391c1 --- /dev/null +++ b/docs/core-gas.rst @@ -0,0 +1,131 @@ +.. _core-accounts: + +============== +Gas Strategies +============== + +Gas strategies are objects that dynamically generate a gas price for a transaction. They can also be used to automatically replace pending transactions within the mempool. + +Gas strategies come in three basic types: + +* **Simple** strategies provide a gas price once, but do not replace pending transactions. +* **Block** strategies provide an initial price, and optionally replace pending transactions based on the number of blocks that have been mined since the first transaction was broadcast. +* **Time** strategies provide an initial price, and optionally replace pending transactions based on the amount of time that has passed since the first transaction was broadcast. + +Using a Gas Strategy +==================== + +To use a gas strategy, first import it from ``brownie.network.gas.strategies``: + +.. code-block:: python + + >>> from brownie.network.gas.strategies import GasNowStrategy + >>> gas_strategy = GasNowStrategy("fast") + +You can then provide the object in the ``gas_price`` field when making a transaction: + +.. code-block:: python + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +When the strategy replaces a pending transaction, the returned :func:`TransactionReceipt ` object will be for the transaction that confirms. + +During :ref:`non-blocking transactions `, all pending transactions are available within the :func:`history ` object. As soon as one transaction confirms, the remaining dropped transactions are removed. + +Setting a Default Gas Strategy +============================== + +You can use :func:`network.gas_price ` to set a gas strategy as the default for all transactions: + +.. code-block:: python + + >>> from brownie.network import gas_price + >>> gas_price(gas_strategy) + +Available Gas Strategies +======================== + +.. py:class:: brownie.network.gas.strategies.LinearScalingStrategy(initial_gas_price, max_gas_price, increment=1.125, time_duration=30) + + Time based scaling strategy for linear gas price increase. + + * ``initial_gas_price``: The initial gas price to use in the first transaction + * ``max_gas_price``: The maximum gas price to use + * ``increment``: Multiplier applied to the previous gas price in order to determine the new gas price + * ``time_duration``: Number of seconds between transactions + + .. code-block:: python + + >>> from brownie.network.gas.strategies import LinearScalingStrategy + >>> gas_strategy = LinearScalingStrategy("10 gwei", "50 gwei", 1.1) + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.ExponentialScalingStrategy(initial_gas_price, max_gas_price, time_duration=30) + + Time based scaling strategy for exponential gas price increase. + + The gas price for each subsequent transaction is calculated as the previous price multiplied by `1.1 ** n` where n is the number of transactions that have been broadcast. In this way the price increase starts gradually and ramps up until confirmation. + + * ``initial_gas_price``: The initial gas price to use in the first transaction + * ``max_gas_price``: The maximum gas price to use + * ``time_duration``: Number of seconds between transactions + + .. code-block:: python + + >>> from brownie.network.gas.strategies import ExponentialScalingStrategy + >>> gas_strategy = ExponentialScalingStrategy("10 gwei", "50 gwei") + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.GasNowStrategy(speed="fast") + + Simple gas strategy for determing a price using the `GasNow `_ API. + + * ``speed``: The gas price to use based on the API call. Options are rapid, fast, standard and slow. + + .. code-block:: python + + >>> from brownie.network.gas.strategies import GasNowStrategy + >>> gas_strategy = GasNowStrategy("fast") + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.GasNowScalingStrategy(initial_speed="standard", max_speed="rapid", increment=1.125, block_duration=2) + + Block based scaling gas strategy using the GasNow API. + + * ``initial_speed``: The initial gas price to use when broadcasting the first transaction. Options are rapid, fast, standard and slow. + * ``max_speed``: The maximum gas price to use when replacing the transaction. Options are rapid, fast, standard and slow. + * ``increment``: A multiplier applied to the most recently used gas price in order to determine the new gas price. If the incremented value is less than or equal to the current ``max_speed`` rate, a new transaction is broadcasted. If the current rate for ``initial_speed`` is greater than the incremented rate, it is used instead. + * ``block_duration``: The number of blocks to wait between broadcasting new transactions. + + .. code-block:: python + + >>> from brownie.network.gas.strategies import GasNowScalingStrategy + >>> gas_strategy = GasNowScalingStrategy("fast", increment=1.2) + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +.. py:class:: brownie.network.gas.strategies.GethMempoolStrategy(position=500, graphql_endpoint=None, block_duration=2) + + Block based scaling gas strategy using Geth's `GraphQL interface `_. + + In order to use this strategy you must be connecting via a Geth node with GraphQL enabled. + + The yielded gas price is determined by sorting transactions in the mempool according to gas price, and returning the price of the transaction at `position`. This is the same technique used by the GasNow API. + + * A position of 200 or less usually places a transaction within the mining block. + * A position of 500 usually places a transaction within the 2nd pending block. + + .. code-block:: python + + >>> from brownie.network.gas.strategies import GethMempoolStrategy + >>> gas_strategy = GethMempoolStrategy(200) + + >>> accounts[0].transfer(accounts[1], "1 ether", gas_price=gas_strategy) + +Building your own Gas Strategy +============================== + +To implement your own gas strategy you must subclass from one of the :ref:`gas strategy abstract base classes `. diff --git a/docs/toctree.rst b/docs/toctree.rst index c70cf36ef..0918bb115 100644 --- a/docs/toctree.rst +++ b/docs/toctree.rst @@ -31,6 +31,7 @@ Brownie core-chain.rst core-transactions.rst core-types.rst + core-gas.rst .. toctree:: From 6ea0c08119fb839cc25cdeac67851b14e57155b1 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 20 Nov 2020 03:36:00 +0400 Subject: [PATCH 23/25] test: basic gas strategy tests --- tests/network/test_gas.py | 66 +++++++++++++++++++++++++ tests/project/compiler/test_solidity.py | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 tests/network/test_gas.py diff --git a/tests/network/test_gas.py b/tests/network/test_gas.py new file mode 100644 index 000000000..ffe6f0363 --- /dev/null +++ b/tests/network/test_gas.py @@ -0,0 +1,66 @@ +import pytest + +from brownie.network.gas.strategies import ExponentialScalingStrategy, LinearScalingStrategy + + +def test_linear_initial(): + strat = LinearScalingStrategy(1, 10) + generator = strat.get_gas_price() + assert next(generator) == 1 + + +def test_linear_max(): + strat = LinearScalingStrategy(100, 1000) + generator = strat.get_gas_price() + last = next(generator) + for i in range(20): + if last == 1000: + assert next(generator) == 1000 + else: + value = next(generator) + assert last < value <= 1000 + last = value + + +@pytest.mark.parametrize("increment", [1.1, 1.25, 1.337, 2]) +def test_linear_increment(increment): + strat = LinearScalingStrategy(100, 100000000000, increment=increment) + generator = strat.get_gas_price() + + last = next(generator) + + for i in range(20): + value = next(generator) + assert int(last * increment) == value + last = value + + +def test_exponential_initial(): + strat = ExponentialScalingStrategy(1, 10) + generator = strat.get_gas_price() + assert next(generator) == 1 + + +def test_exponential_max(): + strat = ExponentialScalingStrategy(100, 1000) + generator = strat.get_gas_price() + last = next(generator) + for i in range(20): + if last == 1000: + assert next(generator) == 1000 + else: + value = next(generator) + assert last < value <= 1000 + last = value + + +def test_exponential_increment(): + strat = ExponentialScalingStrategy(100, 100000000000) + generator = strat.get_gas_price() + + values = [next(generator) for i in range(20)] + + diff = values[1] - values[0] + for i in range(2, 20): + assert values[i] - values[i - 1] > diff + diff = values[i] - values[i - 1] diff --git a/tests/project/compiler/test_solidity.py b/tests/project/compiler/test_solidity.py index af11ccbad..81ee5061e 100644 --- a/tests/project/compiler/test_solidity.py +++ b/tests/project/compiler/test_solidity.py @@ -186,7 +186,7 @@ def test_compile_empty(): def test_get_abi(): - code = "pragma solidity 0.5.7; contract Foo { function baz() external returns (bool); }" + code = "pragma solidity 0.5.0; contract Foo { function baz() external returns (bool); }" abi = compiler.solidity.get_abi(code) assert len(abi) == 1 assert abi["Foo"] == [ From 888fb2dc194d373e6f9a47c4d3a5a05938455c30 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Mon, 23 Nov 2020 02:22:36 +0400 Subject: [PATCH 24/25] fix: raise ValueError if exc is None --- brownie/network/transaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brownie/network/transaction.py b/brownie/network/transaction.py index 1071ab90d..e8b667f12 100644 --- a/brownie/network/transaction.py +++ b/brownie/network/transaction.py @@ -341,7 +341,7 @@ def _raise_if_reverted(self, exc: Any) -> None: return if not web3.supports_traces: # if traces are not available, do not attempt to determine the revert reason - raise exc + raise exc or ValueError("Execution reverted") if self._revert_msg is None: # no revert message and unable to check dev string - have to get trace From 9524cd671b2369536d8394dcdcc7d8b00409b322 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Mon, 23 Nov 2020 19:01:17 +0400 Subject: [PATCH 25/25] refactor: adjust logic for handling failed gasnow api queries --- brownie/network/gas/strategies.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/brownie/network/gas/strategies.py b/brownie/network/gas/strategies.py index 54824e2bf..a890627af 100644 --- a/brownie/network/gas/strategies.py +++ b/brownie/network/gas/strategies.py @@ -1,6 +1,7 @@ import itertools import threading import time +import warnings from typing import Dict, Generator import requests @@ -18,21 +19,24 @@ def _fetch_gasnow(key: str) -> int: global _gasnow_update with _gasnow_lock: - if time.time() - _gasnow_update > 15: - data = None - for i in range(12): + time_since_update = int(time.time() - _gasnow_update) + if time_since_update > 15: + try: response = requests.get( "https://www.gasnow.org/api/v3/gas/price?utm_source=brownie" ) - if response.status_code != 200: - time.sleep(5) - continue + response.raise_for_status() data = response.json()["data"] - break - if data is None: - raise ValueError - _gasnow_update = data.pop("timestamp") // 1000 - _gasnow_data.update(data) + _gasnow_update = data.pop("timestamp") // 1000 + _gasnow_data.update(data) + except requests.exceptions.RequestException as exc: + if time_since_update > 120: + raise + warnings.warn( + f"{type(exc).__name__} while querying GasNow API. " + f"Last successful update was {time_since_update}s ago.", + RuntimeWarning, + ) return _gasnow_data[key]