From eea963ffee134517411b8e74f815bdc390f54542 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Sun, 21 Apr 2024 12:52:33 +0300 Subject: [PATCH 1/7] Adds test for blockheight for all endpoints --- bitcash/network/APIs/BitcoinDotComAPI.py | 7 +++ bitcash/network/APIs/ChaingraphAPI.py | 21 ++++++++ bitcash/network/APIs/__init__.py | 9 ++++ bitcash/network/services.py | 57 ++++++++++++++++++--- tests/network/APIs/test_BitcoinDotComAPI.py | 6 +++ tests/network/APIs/test_ChaingraphAPI.py | 14 +++++ tests/network/test_services.py | 36 +++++++++++++ 7 files changed, 142 insertions(+), 8 deletions(-) diff --git a/bitcash/network/APIs/BitcoinDotComAPI.py b/bitcash/network/APIs/BitcoinDotComAPI.py index f38597e4..9a358a07 100644 --- a/bitcash/network/APIs/BitcoinDotComAPI.py +++ b/bitcash/network/APIs/BitcoinDotComAPI.py @@ -43,6 +43,7 @@ def __init__(self, network_endpoint: str): "address": "address/details/{}", "raw-tx": "rawtransactions/sendRawTransaction", "tx-details": "transaction/details/{}", + "block-height": "blockchain/getBlockCount", } @classmethod @@ -52,6 +53,12 @@ def get_default_endpoints(cls, network): def make_endpoint_url(self, path): return self.network_endpoint + self.PATHS[path] + def get_blockheight(self, *args, **kwargs): + api_url = self.make_endpoint_url("block-height") + r = session.get(api_url) + r.raise_for_status() + return r.json() + def get_balance(self, address, *args, **kwargs): address = cashtokenaddress_to_address(address) api_url = self.make_endpoint_url("address").format(address) diff --git a/bitcash/network/APIs/ChaingraphAPI.py b/bitcash/network/APIs/ChaingraphAPI.py index a07ccfed..547164fe 100644 --- a/bitcash/network/APIs/ChaingraphAPI.py +++ b/bitcash/network/APIs/ChaingraphAPI.py @@ -56,6 +56,27 @@ def send_request(self, json_request, *args, **kwargs): def get_default_endpoints(cls, network): return cls.DEFAULT_ENDPOINTS[network] + def get_blockheight(self, *args, **kwargs): + json_request = { + "query": """ +query GetBlockheight($node: String!) { + block( + limit: 1 + order_by: { height: desc } + where: { accepted_by: { node: { name: { _like: $node } } } } + ) { + height + } +} +""", + "variables": { + "node": self.node_like, + }, + } + json = self.send_request(json_request, *args, **kwargs) + blockheight = int(json["data"]["block"][0]["height"]) + return blockheight + def get_balance(self, address, *args, **kwargs): json_request = { "query": """ diff --git a/bitcash/network/APIs/__init__.py b/bitcash/network/APIs/__init__.py index be4cce0e..25cc4518 100644 --- a/bitcash/network/APIs/__init__.py +++ b/bitcash/network/APIs/__init__.py @@ -23,6 +23,15 @@ def get_default_endpoints(self, network): :rtype: ``list`` of ``str`` """ + @abstractmethod + def get_blockheight(self, *args, **kwargs): + """ + Return the block height. + + :returns: Blockheight + :rtype: ``int`` + """ + @abstractmethod def get_balance(self, address, *args, **kwargs): """ diff --git a/bitcash/network/services.py b/bitcash/network/services.py index 66d71cc3..032bd7bc 100644 --- a/bitcash/network/services.py +++ b/bitcash/network/services.py @@ -122,6 +122,47 @@ class NetworkAPI: requests.exceptions.StreamConsumedError, ) + @classmethod + def get_ordered_endpoints_for(cls, network="mainnet", remove_bad_endpoints=True): + """Gets endpoints ordered by their blockheights. + Solves the problem when an endpoint is stuck on an older block. + + :param network: network in ["mainnet", "testnet", "regtest"]. + :param remove_bad_endpoints: remove unreachable or un-synced endpoints. + """ + endpoints = get_endpoints_for(network) + + endpoints_blockheight = [0 for _ in range(len(endpoints))] + + for i, endpoint in enumerate(endpoints): + try: + endpoints_blockheight[i] = endpoint.get_blockheight( + timeout=DEFAULT_TIMEOUT + ) + except cls.IGNORED_ERRORS: # pragma: no cover + pass + + if sum(endpoints_blockheight) == 0: + raise ConnectionError("All APIs are unreachable.") # pragma: no cover + + ordered_endpoints_blockheight, ordered_endpoints = zip( + *sorted( + zip(endpoints_blockheight, endpoints), + key=lambda tup: tup[0], + reverse=True, + ) + ) + + ordered_endpoints = list(ordered_endpoints) + + if remove_bad_endpoints: + highest_blockheight = ordered_endpoints_blockheight[0] + for i in reversed(range(len(ordered_endpoints_blockheight))): + if ordered_endpoints_blockheight[i] != highest_blockheight: + ordered_endpoints.pop(i) + + return ordered_endpoints + @classmethod def get_balance(cls, address, network="mainnet"): """Gets the balance of an address in satoshi. @@ -131,7 +172,7 @@ def get_balance(cls, address, network="mainnet"): :raises ConnectionError: If all API services fail. :rtype: ``int`` """ - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): try: return endpoint.get_balance(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -148,7 +189,7 @@ def get_transactions(cls, address, network="mainnet"): :raises ConnectionError: If all API services fail. :rtype: ``list`` of ``str`` """ - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): try: return endpoint.get_transactions(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -166,7 +207,7 @@ def get_transaction(cls, txid, network="mainnet"): :rtype: ``Transaction`` """ - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): try: return endpoint.get_transaction(txid, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -186,7 +227,7 @@ def get_tx_amount(cls, txid, txindex, network="mainnet"): :rtype: ``Decimal`` """ - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): try: return endpoint.get_tx_amount(txid, txindex, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -204,7 +245,7 @@ def get_unspent(cls, address, network="mainnet"): :rtype: ``list`` of :class:`~bitcash.network.meta.Unspent` """ - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): try: return endpoint.get_unspent(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -222,7 +263,7 @@ def get_raw_transaction(cls, txid, network="mainnet"): :rtype: ``Transaction`` """ - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): try: return endpoint.get_raw_transaction(txid, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -240,7 +281,7 @@ def broadcast_tx(cls, tx_hex, network="mainnet"): # pragma: no cover """ success = None - for endpoint in get_endpoints_for(network): + for endpoint in cls.get_ordered_endpoints_for(network): _ = [end[0] for end in ChaingraphAPI.get_default_endpoints(network)] if endpoint in _ and network == "mainnet": # Default chaingraph endpoints do not indicate failed broadcast @@ -256,7 +297,7 @@ def broadcast_tx(cls, tx_hex, network="mainnet"): # pragma: no cover if not success: raise ConnectionError( - "Transaction broadcast failed, or " "Unspents were already used." + "Transaction broadcast failed, or Unspents were already used." ) raise ConnectionError("All APIs are unreachable.") diff --git a/tests/network/APIs/test_BitcoinDotComAPI.py b/tests/network/APIs/test_BitcoinDotComAPI.py index d1bb76f9..e952d651 100644 --- a/tests/network/APIs/test_BitcoinDotComAPI.py +++ b/tests/network/APIs/test_BitcoinDotComAPI.py @@ -78,6 +78,12 @@ def setup_method(self): self.monkeypatch = MonkeyPatch() self.api = BitcoinDotComAPI("https://dummy.com/v2/") + def test_get_blockheight(self): + return_json = 800_000 + self.monkeypatch.setattr(_bapi, "session", DummySession(return_json)) + blockheight = self.api.get_blockheight() + assert blockheight == 800_000 + def test_get_balance(self): return_json = { "balanceSat": 2500, diff --git a/tests/network/APIs/test_ChaingraphAPI.py b/tests/network/APIs/test_ChaingraphAPI.py index 1ecff89c..4bb95783 100644 --- a/tests/network/APIs/test_ChaingraphAPI.py +++ b/tests/network/APIs/test_ChaingraphAPI.py @@ -37,6 +37,20 @@ def setup_method(self): self.monkeypatch = MonkeyPatch() self.api = ChaingraphAPI("https://dummy.com/v1/graphql") + def test_get_blockheight(self): + return_json = { + "data": { + "block": [ + { + "height": "123456" + }, + ] + } + } + self.monkeypatch.setattr(_capi, "session", DummySession(return_json)) + blockheight = self.api.get_blockheight() + assert blockheight == 123456 + def test_get_balance(self): return_json = { "data": { diff --git a/tests/network/test_services.py b/tests/network/test_services.py index c345242c..13961f98 100644 --- a/tests/network/test_services.py +++ b/tests/network/test_services.py @@ -3,7 +3,9 @@ import pytest import bitcash +from _pytest.monkeypatch import MonkeyPatch from bitcash.exceptions import InvalidEndpointURLProvided +from bitcash.network import services as _services from bitcash.network.meta import Unspent from bitcash.network.services import ( BitcoinDotComAPI, @@ -72,7 +74,41 @@ def get_raw_transaction(cls, *args, **kwargs): raise_connection_error() +class MockEndpoint: + def __init__(self, blockheight): + self.blockheight = blockheight + + def get_blockheight(self, *args, **kwargs): + if self.blockheight < 0: + raise NetworkAPI.IGNORED_ERRORS[0] + return self.blockheight + + +def mock_get_endpoints_for(network): + return ( + MockEndpoint(4), + MockEndpoint(-1), + MockEndpoint(0), + MockEndpoint(4), + MockEndpoint(4), + MockEndpoint(4), + MockEndpoint(3), + ) + + class TestNetworkAPI: + def test_get_ordered_endpoints_for(self): + monkeypatch = MonkeyPatch() + monkeypatch.setattr(_services, "get_endpoints_for", mock_get_endpoints_for) + endpoints = NetworkAPI.get_ordered_endpoints_for( + network="mainnet", + remove_bad_endpoints=True, + ) + assert len(endpoints) == 4 + # monkeypatch doesn't unset the attribute + # this fails the rest of the tests + monkeypatch.setattr(_services, "get_endpoints_for", get_endpoints_for) + # Mainnet def test_get_balance_mainnet(self): time.sleep(1) From f1e17edf1189ec43e5bba6477ffa1e276a5dc684 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Sun, 21 Apr 2024 13:22:02 +0300 Subject: [PATCH 2/7] Refactors get_sanitized_endpoints_for --- bitcash/network/services.py | 44 ++++++++++++++++------------------ tests/network/test_services.py | 7 +++--- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/bitcash/network/services.py b/bitcash/network/services.py index 032bd7bc..1bd177d7 100644 --- a/bitcash/network/services.py +++ b/bitcash/network/services.py @@ -123,12 +123,11 @@ class NetworkAPI: ) @classmethod - def get_ordered_endpoints_for(cls, network="mainnet", remove_bad_endpoints=True): + def get_sanitized_endpoints_for(cls, network="mainnet"): """Gets endpoints ordered by their blockheights. Solves the problem when an endpoint is stuck on an older block. :param network: network in ["mainnet", "testnet", "regtest"]. - :param remove_bad_endpoints: remove unreachable or un-synced endpoints. """ endpoints = get_endpoints_for(network) @@ -145,23 +144,20 @@ def get_ordered_endpoints_for(cls, network="mainnet", remove_bad_endpoints=True) if sum(endpoints_blockheight) == 0: raise ConnectionError("All APIs are unreachable.") # pragma: no cover - ordered_endpoints_blockheight, ordered_endpoints = zip( - *sorted( - zip(endpoints_blockheight, endpoints), - key=lambda tup: tup[0], - reverse=True, - ) - ) - - ordered_endpoints = list(ordered_endpoints) + # remove unreachable or un-synced endpoints + highest_blockheight = max(endpoints_blockheight) + pop_indices = [] + for i in range(len(endpoints)): + if endpoints_blockheight[i] != highest_blockheight: + pop_indices.append(i) - if remove_bad_endpoints: - highest_blockheight = ordered_endpoints_blockheight[0] - for i in reversed(range(len(ordered_endpoints_blockheight))): - if ordered_endpoints_blockheight[i] != highest_blockheight: - ordered_endpoints.pop(i) + if pop_indices: + endpoints = list(endpoints) + for i in sorted(pop_indices, reverse=True): + endpoints.pop(i) + endpoints = tuple(endpoints) - return ordered_endpoints + return endpoints @classmethod def get_balance(cls, address, network="mainnet"): @@ -172,7 +168,7 @@ def get_balance(cls, address, network="mainnet"): :raises ConnectionError: If all API services fail. :rtype: ``int`` """ - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): try: return endpoint.get_balance(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -189,7 +185,7 @@ def get_transactions(cls, address, network="mainnet"): :raises ConnectionError: If all API services fail. :rtype: ``list`` of ``str`` """ - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): try: return endpoint.get_transactions(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -207,7 +203,7 @@ def get_transaction(cls, txid, network="mainnet"): :rtype: ``Transaction`` """ - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): try: return endpoint.get_transaction(txid, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -227,7 +223,7 @@ def get_tx_amount(cls, txid, txindex, network="mainnet"): :rtype: ``Decimal`` """ - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): try: return endpoint.get_tx_amount(txid, txindex, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -245,7 +241,7 @@ def get_unspent(cls, address, network="mainnet"): :rtype: ``list`` of :class:`~bitcash.network.meta.Unspent` """ - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): try: return endpoint.get_unspent(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -263,7 +259,7 @@ def get_raw_transaction(cls, txid, network="mainnet"): :rtype: ``Transaction`` """ - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): try: return endpoint.get_raw_transaction(txid, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -281,7 +277,7 @@ def broadcast_tx(cls, tx_hex, network="mainnet"): # pragma: no cover """ success = None - for endpoint in cls.get_ordered_endpoints_for(network): + for endpoint in cls.get_sanitized_endpoints_for(network): _ = [end[0] for end in ChaingraphAPI.get_default_endpoints(network)] if endpoint in _ and network == "mainnet": # Default chaingraph endpoints do not indicate failed broadcast diff --git a/tests/network/test_services.py b/tests/network/test_services.py index 13961f98..ba0a7a68 100644 --- a/tests/network/test_services.py +++ b/tests/network/test_services.py @@ -100,11 +100,10 @@ class TestNetworkAPI: def test_get_ordered_endpoints_for(self): monkeypatch = MonkeyPatch() monkeypatch.setattr(_services, "get_endpoints_for", mock_get_endpoints_for) - endpoints = NetworkAPI.get_ordered_endpoints_for( - network="mainnet", - remove_bad_endpoints=True, - ) + endpoints = NetworkAPI.get_sanitized_endpoints_for("mainnet") assert len(endpoints) == 4 + for endpoint in endpoints: + assert endpoint.get_blockheight() == 4 # monkeypatch doesn't unset the attribute # this fails the rest of the tests monkeypatch.setattr(_services, "get_endpoints_for", get_endpoints_for) From 1b5d1853d273aea92d23be499396de2e2efe7296 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Thu, 25 Apr 2024 23:39:34 +0300 Subject: [PATCH 3/7] Adds time_cache for blockheight The cache is invalidated after time-to-live --- bitcash/network/APIs/BitcoinDotComAPI.py | 4 ++- bitcash/network/APIs/ChaingraphAPI.py | 4 ++- bitcash/network/APIs/__init__.py | 4 +++ bitcash/utils.py | 35 ++++++++++++++++++++++++ tests/network/APIs/test_ChaingraphAPI.py | 4 +-- 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/bitcash/network/APIs/BitcoinDotComAPI.py b/bitcash/network/APIs/BitcoinDotComAPI.py index 9a358a07..aaa00bef 100644 --- a/bitcash/network/APIs/BitcoinDotComAPI.py +++ b/bitcash/network/APIs/BitcoinDotComAPI.py @@ -2,10 +2,11 @@ from decimal import Decimal from bitcash.exceptions import InvalidEndpointURLProvided from bitcash.network import currency_to_satoshi -from bitcash.network.APIs import BaseAPI +from bitcash.network.APIs import BaseAPI, DEFAULT_CACHE_TIME from bitcash.network.meta import Unspent from bitcash.network.transaction import Transaction, TxPart from bitcash.format import cashtokenaddress_to_address +from bitcash.utils import time_cache # This class is the interface for Bitcash to interact with # Bitcoin.com based RESTful interfaces. @@ -53,6 +54,7 @@ def get_default_endpoints(cls, network): def make_endpoint_url(self, path): return self.network_endpoint + self.PATHS[path] + @time_cache(DEFAULT_CACHE_TIME) def get_blockheight(self, *args, **kwargs): api_url = self.make_endpoint_url("block-height") r = session.get(api_url) diff --git a/bitcash/network/APIs/ChaingraphAPI.py b/bitcash/network/APIs/ChaingraphAPI.py index 547164fe..f10974e9 100644 --- a/bitcash/network/APIs/ChaingraphAPI.py +++ b/bitcash/network/APIs/ChaingraphAPI.py @@ -1,9 +1,10 @@ from bitcash.network.http import session from bitcash.exceptions import InvalidEndpointURLProvided -from bitcash.network.APIs import BaseAPI +from bitcash.network.APIs import BaseAPI, DEFAULT_CACHE_TIME from bitcash.network.meta import Unspent from bitcash.network.transaction import Transaction, TxPart from bitcash.cashaddress import Address +from bitcash.utils import time_cache class ChaingraphAPI(BaseAPI): @@ -56,6 +57,7 @@ def send_request(self, json_request, *args, **kwargs): def get_default_endpoints(cls, network): return cls.DEFAULT_ENDPOINTS[network] + @time_cache(DEFAULT_CACHE_TIME) def get_blockheight(self, *args, **kwargs): json_request = { "query": """ diff --git a/bitcash/network/APIs/__init__.py b/bitcash/network/APIs/__init__.py index 25cc4518..aa0a8225 100644 --- a/bitcash/network/APIs/__init__.py +++ b/bitcash/network/APIs/__init__.py @@ -1,6 +1,10 @@ from abc import ABC, abstractmethod +# default cache time for the blockheigt retrieval +DEFAULT_CACHE_TIME = 120 + + class BaseAPI(ABC): """ Abstract class for API classes diff --git a/bitcash/utils.py b/bitcash/utils.py index 6fa13fea..89b7dfdc 100644 --- a/bitcash/utils.py +++ b/bitcash/utils.py @@ -1,4 +1,6 @@ import decimal +import functools +import time from binascii import hexlify @@ -68,3 +70,36 @@ def varint_to_int(val): if start_byte == b"\xfd": return int.from_bytes(val.read(2), "little") return int.from_bytes(start_byte, "little") + + +def time_cache(max_age): + """ + Timed cache decorator to store a value until time-to-live + + :param max_age: Time, in seconds, untill when the value is invalidated. + """ + + class ReturnValue: + def __init__(self, value, expiry): + self.value = value + self.expiry = expiry + + def _decorator(fn): + @functools.cache + def cache_fn(*args, **kwargs): + value = fn(*args, **kwargs) + expiry = time.monotonic() + max_age + return ReturnValue(value, expiry) + + @functools.wraps(fn) + def _wrapped(*args, **kwargs): + return_value = cache_fn(*args, **kwargs) + if return_value.expiry < time.monotonic(): + # update the reference to the cache + return_value.value = fn(*args, **kwargs) + return_value.expiry = time.monotonic() + max_age + return return_value.value + + return _wrapped + + return _decorator diff --git a/tests/network/APIs/test_ChaingraphAPI.py b/tests/network/APIs/test_ChaingraphAPI.py index 4bb95783..955eafd9 100644 --- a/tests/network/APIs/test_ChaingraphAPI.py +++ b/tests/network/APIs/test_ChaingraphAPI.py @@ -41,9 +41,7 @@ def test_get_blockheight(self): return_json = { "data": { "block": [ - { - "height": "123456" - }, + {"height": "123456"}, ] } } From f01e14f5b61d977e142cade9b269c849bd754174 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Sun, 5 May 2024 13:39:26 +0300 Subject: [PATCH 4/7] Add cache size to time_cache with LRU cache --- bitcash/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bitcash/utils.py b/bitcash/utils.py index 89b7dfdc..0a9b6665 100644 --- a/bitcash/utils.py +++ b/bitcash/utils.py @@ -72,11 +72,12 @@ def varint_to_int(val): return int.from_bytes(start_byte, "little") -def time_cache(max_age): +def time_cache(max_age: int, cache_size: int = 32): """ Timed cache decorator to store a value until time-to-live :param max_age: Time, in seconds, untill when the value is invalidated. + :param cache_size: Size of LRU cache. """ class ReturnValue: @@ -85,7 +86,7 @@ def __init__(self, value, expiry): self.expiry = expiry def _decorator(fn): - @functools.cache + @functools.lru_cache(maxsize=cache_size) def cache_fn(*args, **kwargs): value = fn(*args, **kwargs) expiry = time.monotonic() + max_age From a7203470a479b992f760c9eae37884bf9500a113 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Sun, 5 May 2024 13:40:03 +0300 Subject: [PATCH 5/7] Refactor rates API to use time cache --- bitcash/network/rates.py | 46 +++++-------------------------------- tests/network/test_rates.py | 34 +++++++++++++++++---------- 2 files changed, 28 insertions(+), 52 deletions(-) diff --git a/bitcash/network/rates.py b/bitcash/network/rates.py index 53ca0405..ef140ba0 100644 --- a/bitcash/network/rates.py +++ b/bitcash/network/rates.py @@ -1,11 +1,9 @@ from collections import OrderedDict from decimal import ROUND_DOWN -from functools import wraps -from time import time import requests from bitcash.network.http import session -from bitcash.utils import Decimal +from bitcash.utils import Decimal, time_cache DEFAULT_CACHE_TIME = 60 @@ -651,42 +649,9 @@ def currency_to_satoshi(amount, currency): return int(satoshis * Decimal(amount)) -class CachedRate: - __slots__ = ("satoshis", "last_update") - - def __init__(self, satoshis, last_update): - self.satoshis = satoshis - self.last_update = last_update - - -def currency_to_satoshi_local_cache(f): - start_time = time() - - cached_rates = dict( - [(currency, CachedRate(None, start_time)) for currency in EXCHANGE_RATES.keys()] - ) - - @wraps(f) - def wrapper(amount, currency): - now = time() - - cached_rate = cached_rates[currency] - - if ( - not cached_rate.satoshis - or now - cached_rate.last_update > DEFAULT_CACHE_TIME - ): - cached_rate.satoshis = EXCHANGE_RATES[currency]() - cached_rate.last_update = now - - return int(cached_rate.satoshis * Decimal(amount)) - - return wrapper - - -@currency_to_satoshi_local_cache -def currency_to_satoshi_local_cached(): - pass # pragma: no cover +@time_cache(max_age=DEFAULT_CACHE_TIME, cache_size=len(EXCHANGE_RATES)) +def _currency_to_satoshi_cached(currency): + return EXCHANGE_RATES[currency]() def currency_to_satoshi_cached(amount, currency): @@ -700,7 +665,8 @@ def currency_to_satoshi_cached(amount, currency): :type currency: ``str`` :rtype: ``int`` """ - return currency_to_satoshi_local_cached(amount, currency) + satoshis = _currency_to_satoshi_cached(currency) + return int(satoshis * Decimal(amount)) def satoshi_to_currency(num, currency): diff --git a/tests/network/test_rates.py b/tests/network/test_rates.py index f0d36a11..8ce433f5 100644 --- a/tests/network/test_rates.py +++ b/tests/network/test_rates.py @@ -1,11 +1,14 @@ from time import sleep, time +from _pytest.monkeypatch import MonkeyPatch import bitcash +from bitcash.network import rates as _rates from bitcash.network.rates import ( RatesAPI, bch_to_satoshi, currency_to_satoshi, currency_to_satoshi_cached, + EXCHANGE_RATES, mbch_to_satoshi, satoshi_to_currency, satoshi_to_currency_cached, @@ -73,38 +76,45 @@ def test_rates_close(): assert rates[-1] / rates[0] < 1.1 and rates[-1] / rates[0] > 0.9 +def _dummy_usd_to_satoshi(): + sleep(1) + return 1 + + +DUMMY_EXCHANGE_RATES = {"usd": _dummy_usd_to_satoshi} + + class TestRateCache: - def test_cache(self): - sleep(0.2) + def setup_method(self): + self.monkeypatch = MonkeyPatch() + def test_cache(self): + self.monkeypatch.setattr(_rates, "EXCHANGE_RATES", DUMMY_EXCHANGE_RATES) start_time = time() - set_rate_cache_time(0) currency_to_satoshi_cached(1, "usd") initial_time = time() - start_time start_time = time() - set_rate_cache_time(60) - currency_to_satoshi_cached(1, "usd") + currency_to_satoshi_cached(2, "usd") cached_time = time() - start_time assert initial_time > cached_time + self.monkeypatch.setattr(_rates, "EXCHANGE_RATES", EXCHANGE_RATES) def test_expires(self): - sleep(0.2) - - set_rate_cache_time(0) + self.monkeypatch.setattr(_rates, "EXCHANGE_RATES", DUMMY_EXCHANGE_RATES) + set_rate_cache_time(1.2) currency_to_satoshi_cached(1, "usd") start_time = time() - set_rate_cache_time(60) - currency_to_satoshi_cached(1, "usd") + currency_to_satoshi_cached(2, "usd") cached_time = time() - start_time sleep(0.2) start_time = time() - set_rate_cache_time(0.1) - currency_to_satoshi_cached(1, "usd") + currency_to_satoshi_cached(3, "usd") update_time = time() - start_time assert update_time > cached_time + self.monkeypatch.setattr(_rates, "EXCHANGE_RATES", EXCHANGE_RATES) From 3ee9bee33cfcb3a37ca3a36c9ddadeb4be5b4f31 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Sun, 5 May 2024 14:23:48 +0300 Subject: [PATCH 6/7] Refactor blockheight caching to get_sanitized_endpoints_for --- bitcash/network/APIs/BitcoinDotComAPI.py | 6 +- bitcash/network/APIs/ChaingraphAPI.py | 4 +- bitcash/network/APIs/__init__.py | 4 - bitcash/network/services.py | 97 +++++++++++++----------- tests/network/test_services.py | 24 +++--- 5 files changed, 69 insertions(+), 66 deletions(-) diff --git a/bitcash/network/APIs/BitcoinDotComAPI.py b/bitcash/network/APIs/BitcoinDotComAPI.py index aaa00bef..a9d47a96 100644 --- a/bitcash/network/APIs/BitcoinDotComAPI.py +++ b/bitcash/network/APIs/BitcoinDotComAPI.py @@ -2,11 +2,10 @@ from decimal import Decimal from bitcash.exceptions import InvalidEndpointURLProvided from bitcash.network import currency_to_satoshi -from bitcash.network.APIs import BaseAPI, DEFAULT_CACHE_TIME +from bitcash.network.APIs import BaseAPI from bitcash.network.meta import Unspent from bitcash.network.transaction import Transaction, TxPart from bitcash.format import cashtokenaddress_to_address -from bitcash.utils import time_cache # This class is the interface for Bitcash to interact with # Bitcoin.com based RESTful interfaces. @@ -54,10 +53,9 @@ def get_default_endpoints(cls, network): def make_endpoint_url(self, path): return self.network_endpoint + self.PATHS[path] - @time_cache(DEFAULT_CACHE_TIME) def get_blockheight(self, *args, **kwargs): api_url = self.make_endpoint_url("block-height") - r = session.get(api_url) + r = session.get(api_url, *args, **kwargs) r.raise_for_status() return r.json() diff --git a/bitcash/network/APIs/ChaingraphAPI.py b/bitcash/network/APIs/ChaingraphAPI.py index f10974e9..547164fe 100644 --- a/bitcash/network/APIs/ChaingraphAPI.py +++ b/bitcash/network/APIs/ChaingraphAPI.py @@ -1,10 +1,9 @@ from bitcash.network.http import session from bitcash.exceptions import InvalidEndpointURLProvided -from bitcash.network.APIs import BaseAPI, DEFAULT_CACHE_TIME +from bitcash.network.APIs import BaseAPI from bitcash.network.meta import Unspent from bitcash.network.transaction import Transaction, TxPart from bitcash.cashaddress import Address -from bitcash.utils import time_cache class ChaingraphAPI(BaseAPI): @@ -57,7 +56,6 @@ def send_request(self, json_request, *args, **kwargs): def get_default_endpoints(cls, network): return cls.DEFAULT_ENDPOINTS[network] - @time_cache(DEFAULT_CACHE_TIME) def get_blockheight(self, *args, **kwargs): json_request = { "query": """ diff --git a/bitcash/network/APIs/__init__.py b/bitcash/network/APIs/__init__.py index aa0a8225..25cc4518 100644 --- a/bitcash/network/APIs/__init__.py +++ b/bitcash/network/APIs/__init__.py @@ -1,10 +1,6 @@ from abc import ABC, abstractmethod -# default cache time for the blockheigt retrieval -DEFAULT_CACHE_TIME = 120 - - class BaseAPI(ABC): """ Abstract class for API classes diff --git a/bitcash/network/services.py b/bitcash/network/services.py index 1bd177d7..dab03647 100644 --- a/bitcash/network/services.py +++ b/bitcash/network/services.py @@ -4,6 +4,7 @@ # Import supported endpoint APIs from bitcash.network.APIs.BitcoinDotComAPI import BitcoinDotComAPI from bitcash.network.APIs.ChaingraphAPI import ChaingraphAPI +from bitcash.utils import time_cache # Dictionary of supported endpoint APIs ENDPOINT_ENV_VARIABLES = { @@ -14,6 +15,9 @@ # Default API call total time timeout DEFAULT_TIMEOUT = 5 +# Default blockheigt cache timeout +DEFAULT_BLOCKHEIGHT_CACHE_TIME = 300 + BCH_TO_SAT_MULTIPLIER = 100000000 NETWORKS = {"mainnet", "testnet", "regtest"} @@ -103,6 +107,48 @@ def get_endpoints_for(network): else: endpoints.append(ENDPOINT_ENV_VARIABLES[endpoint](each)) + return tuple(endpoints) + + +@time_cache(max_age=DEFAULT_BLOCKHEIGHT_CACHE_TIME, cache_size=1) +def _get_endpoints_blockheight(endpoints): + endpoints_blockheight = [0 for _ in range(len(endpoints))] + + for i, endpoint in enumerate(endpoints): + try: + endpoints_blockheight[i] = endpoint.get_blockheight(timeout=DEFAULT_TIMEOUT) + except NetworkAPI.IGNORED_ERRORS: # pragma: no cover + pass + + if sum(endpoints_blockheight) == 0: + raise ConnectionError("All APIs are unreachable.") # pragma: no cover + + return tuple(endpoints_blockheight) + + +def get_sanitized_endpoints_for(network="mainnet"): + """Gets endpoints sanitized by their blockheights. + Solves the problem when an endpoint is stuck on an older block. + + :param network: network in ["mainnet", "testnet", "regtest"]. + """ + endpoints = get_endpoints_for(network) + + endpoints_blockheight = _get_endpoints_blockheight(endpoints) + + # remove unreachable or un-synced endpoints + highest_blockheight = max(endpoints_blockheight) + pop_indices = [] + for i in range(len(endpoints)): + if endpoints_blockheight[i] != highest_blockheight: + pop_indices.append(i) + + if pop_indices: + endpoints = list(endpoints) + for i in sorted(pop_indices, reverse=True): + endpoints.pop(i) + endpoints = tuple(endpoints) + return endpoints @@ -122,43 +168,6 @@ class NetworkAPI: requests.exceptions.StreamConsumedError, ) - @classmethod - def get_sanitized_endpoints_for(cls, network="mainnet"): - """Gets endpoints ordered by their blockheights. - Solves the problem when an endpoint is stuck on an older block. - - :param network: network in ["mainnet", "testnet", "regtest"]. - """ - endpoints = get_endpoints_for(network) - - endpoints_blockheight = [0 for _ in range(len(endpoints))] - - for i, endpoint in enumerate(endpoints): - try: - endpoints_blockheight[i] = endpoint.get_blockheight( - timeout=DEFAULT_TIMEOUT - ) - except cls.IGNORED_ERRORS: # pragma: no cover - pass - - if sum(endpoints_blockheight) == 0: - raise ConnectionError("All APIs are unreachable.") # pragma: no cover - - # remove unreachable or un-synced endpoints - highest_blockheight = max(endpoints_blockheight) - pop_indices = [] - for i in range(len(endpoints)): - if endpoints_blockheight[i] != highest_blockheight: - pop_indices.append(i) - - if pop_indices: - endpoints = list(endpoints) - for i in sorted(pop_indices, reverse=True): - endpoints.pop(i) - endpoints = tuple(endpoints) - - return endpoints - @classmethod def get_balance(cls, address, network="mainnet"): """Gets the balance of an address in satoshi. @@ -168,7 +177,7 @@ def get_balance(cls, address, network="mainnet"): :raises ConnectionError: If all API services fail. :rtype: ``int`` """ - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): try: return endpoint.get_balance(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -185,7 +194,7 @@ def get_transactions(cls, address, network="mainnet"): :raises ConnectionError: If all API services fail. :rtype: ``list`` of ``str`` """ - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): try: return endpoint.get_transactions(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -203,7 +212,7 @@ def get_transaction(cls, txid, network="mainnet"): :rtype: ``Transaction`` """ - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): try: return endpoint.get_transaction(txid, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -223,7 +232,7 @@ def get_tx_amount(cls, txid, txindex, network="mainnet"): :rtype: ``Decimal`` """ - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): try: return endpoint.get_tx_amount(txid, txindex, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -241,7 +250,7 @@ def get_unspent(cls, address, network="mainnet"): :rtype: ``list`` of :class:`~bitcash.network.meta.Unspent` """ - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): try: return endpoint.get_unspent(address, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -259,7 +268,7 @@ def get_raw_transaction(cls, txid, network="mainnet"): :rtype: ``Transaction`` """ - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): try: return endpoint.get_raw_transaction(txid, timeout=DEFAULT_TIMEOUT) except cls.IGNORED_ERRORS: # pragma: no cover @@ -277,7 +286,7 @@ def broadcast_tx(cls, tx_hex, network="mainnet"): # pragma: no cover """ success = None - for endpoint in cls.get_sanitized_endpoints_for(network): + for endpoint in get_sanitized_endpoints_for(network): _ = [end[0] for end in ChaingraphAPI.get_default_endpoints(network)] if endpoint in _ and network == "mainnet": # Default chaingraph endpoints do not indicate failed broadcast diff --git a/tests/network/test_services.py b/tests/network/test_services.py index ba0a7a68..ab5449cb 100644 --- a/tests/network/test_services.py +++ b/tests/network/test_services.py @@ -12,6 +12,7 @@ ChaingraphAPI, NetworkAPI, get_endpoints_for, + get_sanitized_endpoints_for, set_service_timeout, ) from bitcash.network.transaction import Transaction @@ -96,18 +97,19 @@ def mock_get_endpoints_for(network): ) -class TestNetworkAPI: - def test_get_ordered_endpoints_for(self): - monkeypatch = MonkeyPatch() - monkeypatch.setattr(_services, "get_endpoints_for", mock_get_endpoints_for) - endpoints = NetworkAPI.get_sanitized_endpoints_for("mainnet") - assert len(endpoints) == 4 - for endpoint in endpoints: - assert endpoint.get_blockheight() == 4 - # monkeypatch doesn't unset the attribute - # this fails the rest of the tests - monkeypatch.setattr(_services, "get_endpoints_for", get_endpoints_for) +def test_get_ordered_endpoints_for(): + monkeypatch = MonkeyPatch() + monkeypatch.setattr(_services, "get_endpoints_for", mock_get_endpoints_for) + endpoints = get_sanitized_endpoints_for("mainnet") + assert len(endpoints) == 4 + for endpoint in endpoints: + assert endpoint.get_blockheight() == 4 + # monkeypatch doesn't unset the attribute + # this fails the rest of the tests + monkeypatch.setattr(_services, "get_endpoints_for", get_endpoints_for) + +class TestNetworkAPI: # Mainnet def test_get_balance_mainnet(self): time.sleep(1) From 2cbbf30e4ce790af1a1536a264a91c68e163d349 Mon Sep 17 00:00:00 2001 From: "Yashasvi S. Ranawat" Date: Sun, 5 May 2024 16:26:07 +0300 Subject: [PATCH 7/7] Refactor caching to cache sanitized endpoints Improves implementation speed drastically. --- bitcash/network/services.py | 28 +++++++++++----------------- tests/network/test_services.py | 2 +- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/bitcash/network/services.py b/bitcash/network/services.py index dab03647..374b8e84 100644 --- a/bitcash/network/services.py +++ b/bitcash/network/services.py @@ -15,8 +15,8 @@ # Default API call total time timeout DEFAULT_TIMEOUT = 5 -# Default blockheigt cache timeout -DEFAULT_BLOCKHEIGHT_CACHE_TIME = 300 +# Default sanitized endpoint, based on blockheigt, cache timeout +DEFAULT_SANITIZED_ENDPOINTS_CACHE_TIME = 300 BCH_TO_SAT_MULTIPLIER = 100000000 @@ -110,8 +110,15 @@ def get_endpoints_for(network): return tuple(endpoints) -@time_cache(max_age=DEFAULT_BLOCKHEIGHT_CACHE_TIME, cache_size=1) -def _get_endpoints_blockheight(endpoints): +@time_cache(max_age=DEFAULT_SANITIZED_ENDPOINTS_CACHE_TIME, cache_size=len(NETWORKS)) +def get_sanitized_endpoints_for(network="mainnet"): + """Gets endpoints sanitized by their blockheights. + Solves the problem when an endpoint is stuck on an older block. + + :param network: network in ["mainnet", "testnet", "regtest"]. + """ + endpoints = get_endpoints_for(network) + endpoints_blockheight = [0 for _ in range(len(endpoints))] for i, endpoint in enumerate(endpoints): @@ -123,19 +130,6 @@ def _get_endpoints_blockheight(endpoints): if sum(endpoints_blockheight) == 0: raise ConnectionError("All APIs are unreachable.") # pragma: no cover - return tuple(endpoints_blockheight) - - -def get_sanitized_endpoints_for(network="mainnet"): - """Gets endpoints sanitized by their blockheights. - Solves the problem when an endpoint is stuck on an older block. - - :param network: network in ["mainnet", "testnet", "regtest"]. - """ - endpoints = get_endpoints_for(network) - - endpoints_blockheight = _get_endpoints_blockheight(endpoints) - # remove unreachable or un-synced endpoints highest_blockheight = max(endpoints_blockheight) pop_indices = [] diff --git a/tests/network/test_services.py b/tests/network/test_services.py index ab5449cb..7e8d4c90 100644 --- a/tests/network/test_services.py +++ b/tests/network/test_services.py @@ -100,7 +100,7 @@ def mock_get_endpoints_for(network): def test_get_ordered_endpoints_for(): monkeypatch = MonkeyPatch() monkeypatch.setattr(_services, "get_endpoints_for", mock_get_endpoints_for) - endpoints = get_sanitized_endpoints_for("mainnet") + endpoints = get_sanitized_endpoints_for("mock_mainnet") assert len(endpoints) == 4 for endpoint in endpoints: assert endpoint.get_blockheight() == 4