Add withdrawals cache

deploy/gnosis/docker-compose.yml

@@ -24,7 +24,7 @@ networks:
 services:
   oracle:
     container_name: oracle_gnosis
-    image: europe-west4-docker.pkg.dev/stakewiselabs/public/oracle:v3.0.0
+    image: europe-west4-docker.pkg.dev/stakewiselabs/public/oracle:v4.0.0
     restart: always
     entrypoint: ["python"]
     command: ["oracle/oracle/main.py"]
@@ -34,7 +34,7 @@ services:
 
   keeper:
     container_name: keeper_gnosis
-    image: europe-west4-docker.pkg.dev/stakewiselabs/public/oracle:v3.0.0
+    image: europe-west4-docker.pkg.dev/stakewiselabs/public/oracle:v4.0.0
     restart: always
     entrypoint: ["python"]
     command: ["oracle/keeper/main.py"]

oracle/networks.py

@@ -97,6 +97,8 @@
             Web3.toChecksumAddress("0x59ecf48345a221e0731e785ed79ed40d0a94e2a5"): 63,
             Web3.toChecksumAddress("0x01f26d7f195a37d368cb772ed75ef70dd29700f5"): 64,
         },
+        WITHDRAWALS_CACHE_BLOCK=None,
+        WITHDRAWALS_CACHE_AMOUNT=None,
     ),
     HARBOUR_MAINNET: dict(
         STAKEWISE_SUBGRAPH_URLS=config(
@@ -174,6 +176,8 @@
         IS_POA=False,
         DEPOSIT_TOKEN_SYMBOL="ETH",
         VALIDATORS_SPLIT={},
+        WITHDRAWALS_CACHE_BLOCK=None,
+        WITHDRAWALS_CACHE_AMOUNT=None,
     ),
     GOERLI: dict(
         STAKEWISE_SUBGRAPH_URLS=config(
@@ -248,6 +252,8 @@
         IS_POA=True,
         DEPOSIT_TOKEN_SYMBOL="ETH",
         VALIDATORS_SPLIT={},
+        WITHDRAWALS_CACHE_BLOCK=None,
+        WITHDRAWALS_CACHE_AMOUNT=None,
     ),
     HARBOUR_GOERLI: dict(
         STAKEWISE_SUBGRAPH_URLS=config(
@@ -325,6 +331,8 @@
         IS_POA=True,
         DEPOSIT_TOKEN_SYMBOL="ETH",
         VALIDATORS_SPLIT={},
+        WITHDRAWALS_CACHE_BLOCK=None,
+        WITHDRAWALS_CACHE_AMOUNT=None,
     ),
     GNOSIS_CHAIN: dict(
         STAKEWISE_SUBGRAPH_URLS=config(
@@ -402,5 +410,7 @@
             Web3.toChecksumAddress("0x59ecf48345a221e0731e785ed79ed40d0a94e2a5"): 4971,
             Web3.toChecksumAddress("0xf37c8f35fc820354b402054699610c098559ae44"): 4971,
         },
+        WITHDRAWALS_CACHE_BLOCK=33192932,
+        WITHDRAWALS_CACHE_AMOUNT=286633425704392,
     ),
 }

oracle/oracle/main.py

@@ -17,7 +17,7 @@
 )
 from oracle.oracle.distributor.controller import DistributorController
 from oracle.oracle.health_server import oracle_routes
-from oracle.oracle.rewards.controller import RewardsController
+from oracle.oracle.rewards.controller import RewardsController, WithdrawalsCache
 from oracle.oracle.rewards.eth2 import get_finality_checkpoints, get_genesis
 from oracle.oracle.validators.controller import ValidatorsController
 from oracle.oracle.vote import submit_vote
@@ -56,11 +56,15 @@ async def main() -> None:
 
     # fetch ETH2 genesis
     genesis = await get_genesis(session)
-
+    withdrawals_cache = WithdrawalsCache(
+        NETWORK_CONFIG["WITHDRAWALS_CACHE_BLOCK"],
+        NETWORK_CONFIG["WITHDRAWALS_CACHE_AMOUNT"],
+    )
     rewards_controller = RewardsController(
         aiohttp_session=session,
         genesis_timestamp=int(genesis["genesis_time"]),
         oracle=oracle_account,
+        withdrawals_cache=withdrawals_cache,
     )
     distributor_controller = DistributorController(oracle_account)
     validators_controller = ValidatorsController(oracle_account)

oracle/oracle/rewards/controller.py

@@ -43,6 +43,19 @@
 w3 = Web3()
 
 
+class WithdrawalsCache:
+    def __init__(self, block: BlockNumber = None, withdrawals: Wei = None):
+        self.block = block
+        self.withdrawals = withdrawals
+
+    def set(self, block: BlockNumber, withdrawals: Wei):
+        self.block = block
+        self.withdrawals = withdrawals
+
+    def get(self) -> tuple[BlockNumber, Wei]:
+        return self.block, self.withdrawals
+
+
 class RewardsController(object):
     """Updates total rewards and activated validators number."""
 
@@ -51,6 +64,7 @@ def __init__(
         aiohttp_session: ClientSession,
         genesis_timestamp: int,
         oracle: LocalAccount,
+        withdrawals_cache: WithdrawalsCache,
     ) -> None:
         self.deposit_amount: Wei = Web3.toWei(32, "ether")
         self.aiohttp_session = aiohttp_session
@@ -64,6 +78,7 @@ def __init__(
         self.deposit_token_symbol = NETWORK_CONFIG["DEPOSIT_TOKEN_SYMBOL"]
         self.withdrawals_genesis_epoch = NETWORK_CONFIG["WITHDRAWALS_GENESIS_EPOCH"]
         self.last_vote_total_rewards = None
+        self.withdrawals_cache = withdrawals_cache
 
     @save
     async def process(
@@ -216,6 +231,18 @@ async def calculate_withdrawal_rewards(
         if not from_block or from_block >= to_block:
             return Wei(0)
 
+        logger.info(
+            f"Calculating pool validator withdrawals "
+            f"from block: {from_block} to block: {to_block}"
+        )
+        cached_block, cached_withdrawals = self.withdrawals_cache.get()
+        if cached_block is not None:
+            logger.info(
+                f"Restored cached {cached_withdrawals} withdrawals on block: {cached_block}"
+            )
+            from_block = cached_block
+            withdrawals_amount += cached_withdrawals
+
         logger.info(
             f"Retrieving pool validator withdrawals "
             f"from block: {from_block} to block: {to_block}"
@@ -235,6 +262,8 @@ async def calculate_withdrawal_rewards(
         if NETWORK == GNOSIS_CHAIN:
             # apply mGNO <-> GNO exchange rate
             withdrawals_amount = Wei(int(withdrawals_amount * WAD // MGNO_RATE))
+
+        self.withdrawals_cache.set(to_block, withdrawals_amount)
         return withdrawals_amount
 
     async def fetch_withdrawal_chunk(

oracle/oracle/rewards/tests/test_controller.py

@@ -7,7 +7,7 @@
 from oracle.oracle.tests.common import get_test_oracle
 from oracle.oracle.tests.factories import faker
 
-from ..controller import RewardsController
+from ..controller import RewardsController, WithdrawalsCache
 from ..types import RewardsVotingParameters, Withdrawal
 
 epoch = faker.random_int(150000, 250000)
@@ -106,6 +106,7 @@ async def test_process_success(self):
                 aiohttp_session=session,
                 genesis_timestamp=1606824023,
                 oracle=get_test_oracle(),
+                withdrawals_cache=WithdrawalsCache(None, None),
             )
             await controller.process(
                 voting_params=RewardsVotingParameters(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "oracle"
-version = "3.1.0"
+version = "4.0.0"
 description = "StakeWise Oracles are responsible for submitting off-chain data."
 authors = ["Dmitri Tsumak <[email protected]>"]
 license = "AGPL-3.0-only"