From ce93aea845f0ae50de4034dd048ed0d607cc3bc4 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 6 Jun 2019 16:36:26 -0700 Subject: [PATCH] Add a maximum batch size for regular sync imports Without this maximum batch size, when the first batch takes a long time to execute, the second batch will be enormous. Perhaps most importantly, the back-pressure set by _db_buffer_capacity is never triggered if the batch is unlimited size. --- trinity/sync/full/chain.py | 7 +++++-- trinity/sync/full/constants.py | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/trinity/sync/full/chain.py b/trinity/sync/full/chain.py index 8d779dc18d..d318bdf6f3 100644 --- a/trinity/sync/full/chain.py +++ b/trinity/sync/full/chain.py @@ -77,6 +77,7 @@ from trinity.sync.full.constants import ( HEADER_QUEUE_SIZE_TARGET, BLOCK_QUEUE_SIZE_TARGET, + BLOCK_IMPORT_QUEUE_SIZE_TARGET, ) from trinity._utils.datastructures import ( BaseOrderedTaskPreparation, @@ -1040,8 +1041,10 @@ async def _import_ready_blocks(self) -> None: while self.is_operational: timer = Timer() - # wait for block bodies to become ready for execution - completed_headers = await self.wait(self._block_import_tracker.ready_tasks()) + # This tracker waits for all prerequisites to be complete, and returns headers in + # order, so that each header's parent is already persisted. + get_ready_coro = self._block_import_tracker.ready_tasks(BLOCK_IMPORT_QUEUE_SIZE_TARGET) + completed_headers = await self.wait(get_ready_coro) if self._block_import_tracker.has_ready_tasks(): # Even after clearing out a big batch, there is no available capacity, so diff --git a/trinity/sync/full/constants.py b/trinity/sync/full/constants.py index da4109dfa9..c1166c3c45 100644 --- a/trinity/sync/full/constants.py +++ b/trinity/sync/full/constants.py @@ -16,3 +16,10 @@ # How many blocks to persist at a time # Only need a few seconds of buffer on the DB write side. BLOCK_QUEUE_SIZE_TARGET = 1000 + +# How many blocks to import at a time +# Only need a few seconds of buffer on the DB side +# This is specifically for blocks where execution happens locally. +# So each block might have a pretty significant execution time, on +# the order of seconds. +BLOCK_IMPORT_QUEUE_SIZE_TARGET = 10