random 500 internal error and RemoteDisconnected

[adradr]

Describe the bug
I am constantly getting 500 internal error responses for get_vault_assets_balance. Other times I am also receiving:

requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

To Reproduce
Please see below

Expected behavior
I am trying to make a snapshot of our Fireblocks account for backup and also accounting purposes. I am trying to iterate over each vault and its assets and gathering balances for each.

Versions (please complete the following information):

• Python Version: 3.12.1
• fireblocks-sdk version: fireblocks_sdk==2.5.1

Additional context
Here is an excerpt of my related code snippets and logs:

def catch_and_retry_fireblocks_exception(func):
    @functools.wraps(func)
    def wrapper(self, *args, **kwargs):
        max_retries = 3
        retries = 0
        while retries < max_retries:
            try:
                return func(self, *args, **kwargs)
            except FireblocksApiException as e:
                self.logger.error(
                    f"[Fireblocks] Error occurred for function {func.__name__}: {e}"
                )
                self.logger.error(f"[Fireblocks] Arguments: {args}, {kwargs}")
                # Handle the exception as needed
                retries += 1
                if retries < max_retries:
                    # Wait for 1 second before retrying
                    time.sleep(0.1)
                    self.logger.info(f"[Fireblocks] Retrying {func.__name__}...")
                else:
                    self.logger.warning(
                        f"[Fireblocks] Maximum retries reached. Skipping {func.__name__}."
                    )

    return wrapper


class FireblocksSDKWrapper:

    def __init__(self, api_key: str, private_key: str):
        self.fireblocks = FireblocksSDK(api_key=api_key, private_key=private_key)
        self.logger = Logger(name=__name__, level="INFO").get_logger()

    @catch_and_retry_fireblocks_exception
    def get_vault_accounts_paginated(self):
        next_page = True
        paged_filter = PagedVaultAccountsRequestFilters()
        accounts = []
        while next_page:
            vaults = self.fireblocks.get_vault_accounts_with_page_info(paged_filter)
            accounts = vaults["accounts"]
            accounts.extend(accounts)

            # Paginate through accounts
            if vaults.get("paging").get("after") is None:
                next_page = False
            else:
                paged_filter.after = vaults.get("paging").get("after")

        return accounts

    @catch_and_retry_fireblocks_exception
    def get_internal_wallets(self):
        return self.fireblocks.get_internal_wallets()

    @catch_and_retry_fireblocks_exception
    def get_deposit_addresses(self, vault_account_id: typing.Any, asset_id: typing.Any):
        return self.fireblocks.get_deposit_addresses(vault_account_id, asset_id)

    @catch_and_retry_fireblocks_exception
    def get_vault_assets_balance(
        self,
        account_name_prefix: typing.Any | None = None,
        account_name_suffix: typing.Any | None = None,
    ):
        return self.fireblocks.get_vault_assets_balance(
            account_name_prefix,
            account_name_suffix,
        )


@dataclasses.dataclass
class FireblocksAccountingLoader(ExchangeLoaderBase):
    fireblocks: typing.Optional[FireblocksSDKWrapper] = dataclasses.field(init=True)
    vault_accounts: typing.List[dict] = dataclasses.field(init=False)
    internal_wallets: typing.List[dict] = dataclasses.field(init=False)
    accounts: typing.List[dict] = dataclasses.field(init=False)
    logger: logging.Logger = dataclasses.field(
        default_factory=lambda: logging.Logger(__name__)
    )

    def __post_init__(self):
        self.vault_accounts = self.fireblocks.get_vault_accounts_paginated()
        self.internal_wallets = self.fireblocks.get_internal_wallets()
        self.accounts = self.vault_accounts  # + self.internal_wallets

    def fetch_accounts(self, utc_timestamp: str) -> pd.DataFrame:
        df = pd.DataFrame(self.accounts)

        # explode the assets column
        df = df.explode("assets")

        # Cast nan assets to {}
        df.assets = df.assets.fillna({i: {} for i in df.index})

        # create new columns for each key in the dictionaries
        df["asset_id"] = df["assets"].apply(lambda x: x["id"] if "id" in x else None)
        df["asset_total"] = df["assets"].apply(
            lambda x: x["total"] if "total" in x else None
        )
        df["asset_balance"] = df["assets"].apply(
            lambda x: x["balance"] if "balance" in x else None
        )
        df["account_type"] = df["name"].apply(
            lambda x: (
                "internal_wallet" if x == "Gas Station Wallet" else "vault_account"
            )
        )

        # Cast id to int
        df["id"] = df["id"].astype(int)

        # Remove "Network Deposits" vault accounts
        df = df[df.name != "Network Deposits"]

        # Add the timestamp to the dataframe
        df["timestamp"] = utc_timestamp
        # drop the original assets column
        df = df.drop("assets", axis=1)
        # Reset index
        df = df.reset_index(drop=True)

        # Fetch asset_id to a dict

        # Get the vault account asset information
        for row, col in df.iterrows():
            # Skip if the account is an internal wallet
            if col.account_type == "internal_wallet":
                # Get an index of the internal wallet
                internal_wallet_index = [
                    i for i, x in enumerate(self.internal_wallets) if x["id"] == col.id
                ][0]
                asset_info = self.internal_wallets[internal_wallet_index]["assets"]
            elif col.account_type == "vault_account" and col.asset_id is not None:
                asset_info = self.fireblocks.get_deposit_addresses(
                    vault_account_id=col.id,
                    asset_id=col.asset_id,
                )
            else:
                asset_info = []
            # Restructure the asset_info into a dictionary
            asset_dict = {}
            for asset in asset_info:
                asset_dict[asset["address"]] = {
                    "asset_id": asset["assetId"] if "assetId" in asset else None,
                    "address": asset["address"] if "address" in asset else None,
                    "tag": asset["tag"] if "tag" in asset else None,
                    "description": (
                        asset["description"] if "description" in asset else None
                    ),
                    "type": asset["type"] if "type" in asset else None,
                    "addressFormat": (
                        asset["addressFormat"] if "addressFormat" in asset else None
                    ),
                    "legacyAddress": (
                        asset["legacyAddress"] if "legacyAddress" in asset else None
                    ),
                    "bip44AddressIndex": (
                        asset["bip44AddressIndex"]
                        if "bip44AddressIndex" in asset
                        else None
                    ),
                }
            # Add asset_info as a column to the dataframe being iterated over
            df.loc[row, "addresses"] = [[asset_dict]]

        # Get additional balance information
        df_assets = pd.DataFrame()
        for vault_name in df.name.unique():
            # Skip if the account is an internal wallet
            if vault_name == "Gas Station Wallet":
                continue
            asset_balances = pd.DataFrame(
                self.fireblocks.get_vault_assets_balance(vault_name)
            )
            asset_balances = asset_balances.rename(columns={"id": "asset_id"})
            asset_balances["name"] = vault_name
            df_assets = pd.concat([df_assets, asset_balances], axis=0)

        # Merge the asset balances with the dataframe
        df_merged = pd.merge(df, df_assets, on=["name", "asset_id"])

        # Calculate prices
        asset_ids = df_merged.asset_id.unique()
        asset_prices = {}
        for asset_id in asset_ids:
            try:
                asset_id_split = asset_id.split("_")[0]
                asset_prices[asset_id] = self.calculate_usd_price(
                    asset=asset_id_split,
                    exchange=ccxt.bitmart(),  # Using bitmart since only they have RND
                )
            except Exception as e:  # pylint: disable=broad-except
                self.logger.info(
                    "[Fireblocks] Could not fetch price for %s (error: %s)",
                    asset_id,
                    e,
                )
                continue

        for row, col in df_merged.iterrows():
            try:
                asset_price = asset_prices[col.asset_id]
                df_merged.loc[row, "usd_price"] = asset_prices[col.asset_id]
                df_merged.loc[row, "usd_value"] = (
                    col.asset_total * asset_price if asset_price is not None else None
                )
            except Exception as e:  # pylint: disable=broad-except
                self.logger.info(
                    "[Fireblocks] Could not set price for %s: %s)", col.asset_id, e
                )
                continue

        return df_merged

    def to_dataframe(self, utc_timestamp: pd.Timestamp = None):
        return self.fetch_accounts(
            utc_timestamp=(
                pd.Timestamp.utcnow() if utc_timestamp is None else utc_timestamp
            )
        )

2024-03-07 10:56:19,315 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 10:56:19,426 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:08:53,224 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:08:53,327 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:11:35,880 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:11:35,984 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:14:23,272 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:14:23,378 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:17:10,825 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:17:10,932 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:19:52,531 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:19:52,640 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:22:33,295 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:22:33,401 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:25:17,782 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:25:17,890 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:28:00,107 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500

2024-03-07 11:28:00,211 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...

Exception has occurred: ConnectionError       (note: full exception trace is shown but execution is paused at: <module>)
('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
http.client.RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

During handling of the above exception, another exception occurred:

  File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/utils/fireblocks.py", line 74, in get_vault_assets_balance
    return self.fireblocks.get_vault_assets_balance(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/utils/fireblocks.py", line 17, in wrapper
    return func(self, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/data/loaders.py", line 316, in fetch_accounts
    self.fireblocks.get_vault_assets_balance(vault_name)
  File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/data/loaders.py", line 357, in to_dataframe
    return self.fetch_accounts(
           ^^^^^^^^^^^^^^^^^^^^
  File "/var/folders/w6/dh79cy2x0sjdkb2y4vrs8kq00000gn/T/ipykernel_18952/3364672574.py", line 1, in <module> (Current frame)
    loader.to_dataframe()
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))