Eos challenge optimization (#15642)

* fix cache miss (#107)

* Increase vdf client timeout (#103)

* Increase vdf client timeout.

* Reset only on new peak.

* Increase next_iters_count to 10 (#118)

* new end of slot ses validation (#99)

* eos validate infused challenge chain hash in all locations, validate deficit

* lint

* add iterations/diff check

* check all cases of ses

* remove asserts log and return none

* remove asserts log and return none

* typo

* fix store tests

* fix store tests

* lint

* remove logs

---------

Co-authored-by: Almog De Paz <[email protected]>
Co-authored-by: Florin Chirica <[email protected]>
Co-authored-by: Earle Lowe <[email protected]>

chia/consensus/get_block_challenge.py

@@ -95,6 +95,8 @@ def get_block_challenge(
                 if curr.first_in_sub_slot:
                     assert curr.finished_challenge_slot_hashes is not None
                     reversed_challenge_hashes += reversed(curr.finished_challenge_slot_hashes)
+                    if len(reversed_challenge_hashes) >= challenges_to_look_for:
+                        break
                 if curr.height == 0:
                     assert curr.finished_challenge_slot_hashes is not None
                     assert len(curr.finished_challenge_slot_hashes) > 0

chia/full_node/full_node.py

@@ -1455,6 +1455,8 @@ async def peak_post_processing(
             sub_slots[1],
             fork_block,
             self.blockchain,
+            sub_slot_iters,
+            difficulty,
         )
 
         if fns_peak_result.new_signage_points is not None and peer is not None:
@@ -2138,6 +2140,8 @@ async def add_end_of_sub_slot(
                 end_of_slot_bundle,
                 self.blockchain,
                 peak,
+                next_sub_slot_iters,
+                next_difficulty,
                 await self.blockchain.get_full_peak(),
             )
             # It may be an empty list, even if it's not None. Not None means added successfully

chia/full_node/full_node_store.py

@@ -10,15 +10,14 @@
 from chia.consensus.blockchain_interface import BlockchainInterface
 from chia.consensus.constants import ConsensusConstants
 from chia.consensus.difficulty_adjustment import can_finish_sub_and_full_epoch
-from chia.consensus.make_sub_epoch_summary import next_sub_epoch_summary
+from chia.consensus.make_sub_epoch_summary import make_sub_epoch_summary
 from chia.consensus.multiprocess_validation import PreValidationResult
 from chia.consensus.pot_iterations import calculate_sp_interval_iters
 from chia.full_node.signage_point import SignagePoint
 from chia.protocols import timelord_protocol
 from chia.server.outbound_message import Message
 from chia.types.blockchain_format.classgroup import ClassgroupElement
 from chia.types.blockchain_format.sized_bytes import bytes32
-from chia.types.blockchain_format.sub_epoch_summary import SubEpochSummary
 from chia.types.blockchain_format.vdf import VDFInfo
 from chia.types.end_of_slot_bundle import EndOfSubSlotBundle
 from chia.types.full_block import FullBlock
@@ -256,6 +255,8 @@ def new_finished_sub_slot(
         eos: EndOfSubSlotBundle,
         blocks: BlockchainInterface,
         peak: Optional[BlockRecord],
+        next_sub_slot_iters: uint64,
+        next_difficulty: uint64,
         peak_full_block: Optional[FullBlock],
     ) -> Optional[List[timelord_protocol.NewInfusionPointVDF]]:
         """
@@ -284,6 +285,11 @@ def new_finished_sub_slot(
         if eos.challenge_chain.challenge_chain_end_of_slot_vdf.challenge != cc_challenge:
             # This slot does not append to our next slot
             # This prevent other peers from appending fake VDFs to our cache
+            log.error(
+                f"bad cc_challenge in new_finished_sub_slot, "
+                f"got {eos.challenge_chain.challenge_chain_end_of_slot_vdf.challenge}"
+                f"expected {cc_challenge}"
+            )
             return None
 
         if peak is None:
@@ -301,6 +307,7 @@ def new_finished_sub_slot(
             # the finished subslot, and the peak is not fully added yet, so it looks like we still need the subslot.
             # In that case, we will exit here and let the new_peak code add the subslot.
             if total_iters < peak.total_iters:
+                log.debug("dont add slot, total_iters < peak.total_iters")
                 return None
 
             rc_challenge = eos.reward_chain.end_of_slot_vdf.challenge
@@ -315,6 +322,17 @@ def new_finished_sub_slot(
                 log.info(f"Don't have challenge hash {rc_challenge}, caching EOS")
                 return None
 
+            if peak.deficit == 0:
+                if eos.reward_chain.deficit != self.constants.MIN_BLOCKS_PER_CHALLENGE_BLOCK:
+                    log.error(
+                        f"eos reward_chain deficit got {eos.reward_chain.deficit} "
+                        f"expected {self.constants.MIN_BLOCKS_PER_CHALLENGE_BLOCK}"
+                    )
+                    return None
+            elif eos.reward_chain.deficit != peak.deficit:
+                log.error(f"wrong eos reward_chain deficit got {eos.reward_chain.deficit} expected {peak.deficit}")
+                return None
+
             if peak.deficit == self.constants.MIN_BLOCKS_PER_CHALLENGE_BLOCK:
                 icc_start_element = None
             elif peak.deficit == self.constants.MIN_BLOCKS_PER_CHALLENGE_BLOCK - 1:
@@ -335,27 +353,67 @@ def new_finished_sub_slot(
                     icc_iters = sub_slot_iters
                 assert icc_challenge is not None
 
-            if can_finish_sub_and_full_epoch(
+            finish_se, finish_epoch = can_finish_sub_and_full_epoch(
                 self.constants,
                 blocks,
                 peak.height,
                 peak.prev_hash,
                 peak.deficit,
                 peak.sub_epoch_summary_included is not None,
-            )[0]:
-                assert peak_full_block is not None
-                ses: Optional[SubEpochSummary] = next_sub_epoch_summary(
-                    self.constants, blocks, peak.required_iters, peak_full_block, True
+            )
+            if finish_se:
+                # this is the first slot in a new sub epoch, should include SES
+                expected_sub_epoch_summary = make_sub_epoch_summary(
+                    self.constants,
+                    blocks,
+                    peak.height,
+                    blocks.block_record(blocks.block_record(peak.prev_hash).prev_hash),
+                    next_difficulty if finish_epoch else None,
+                    next_sub_slot_iters if finish_epoch else None,
                 )
-                if ses is not None:
-                    if eos.challenge_chain.subepoch_summary_hash != ses.get_hash():
-                        log.warning(f"SES not correct {ses.get_hash(), eos.challenge_chain}")
+
+                if eos.challenge_chain.subepoch_summary_hash is None:
+                    log.warning("SES should not be None")
+                    return None
+
+                if eos.challenge_chain.subepoch_summary_hash != expected_sub_epoch_summary.get_hash():
+                    log.warning(
+                        f"Bad SES, expected {expected_sub_epoch_summary} "
+                        f"expected hash {expected_sub_epoch_summary.get_hash()}, got {eos.challenge_chain}"
+                    )
+                    return None
+
+                if finish_epoch:
+                    # this is the first slot in a new epoch check diff and iterations
+                    if (
+                        eos.challenge_chain.new_sub_slot_iters is None
+                        or eos.challenge_chain.new_sub_slot_iters != next_sub_slot_iters
+                    ):
+                        log.error("wrong new iterations at end of slot bundle")
                         return None
+
+                    if (
+                        eos.challenge_chain.new_difficulty is None
+                        or eos.challenge_chain.new_difficulty != next_difficulty
+                    ):
+                        log.info("wrong new difficulty at end of slot bundle")
+                        return None
+
                 else:
-                    if eos.challenge_chain.subepoch_summary_hash is not None:
-                        log.warning("SES not correct, should be None")
+                    if eos.challenge_chain.new_sub_slot_iters is not None:
+                        log.error("got new iterations at end of slot bundle when it should be None")
+                        return None
+
+                    if eos.challenge_chain.new_difficulty is not None:
+                        log.info("got new difficulty at end of slot bundle when it should be None")
                         return None
+
         else:
+            # empty slots dont have sub_epoch_summary
+            if eos.challenge_chain.subepoch_summary_hash is not None:
+                log.warning("SES not correct, should be None in an empty slot")
+                return None
+
             # This is on an empty slot
             cc_start_element = ClassgroupElement.get_default_element()
             icc_start_element = ClassgroupElement.get_default_element()
@@ -417,6 +475,14 @@ def new_finished_sub_slot(
             assert eos.infused_challenge_chain is not None
             assert eos.infused_challenge_chain is not None
             assert eos.proofs.infused_challenge_chain_slot_proof is not None
+            if eos.reward_chain.deficit == self.constants.MIN_BLOCKS_PER_CHALLENGE_BLOCK:
+                # only at the end of a challenge slot
+                if eos.infused_challenge_chain.get_hash() != eos.challenge_chain.infused_challenge_chain_sub_slot_hash:
+                    log.error("infused_challenge_chain mismatch in challenge_chain")
+                    return None
+            else:
+                assert eos.challenge_chain.infused_challenge_chain_sub_slot_hash is None
+            assert eos.infused_challenge_chain.get_hash() == eos.reward_chain.infused_challenge_chain_sub_slot_hash
 
             partial_icc_vdf_info = VDFInfo(
                 icc_challenge,
@@ -449,6 +515,10 @@ def new_finished_sub_slot(
             # This is the first sub slot and it's empty, therefore there is no ICC
             if eos.infused_challenge_chain is not None or eos.proofs.infused_challenge_chain_slot_proof is not None:
                 return None
+            if eos.challenge_chain.infused_challenge_chain_sub_slot_hash is not None:
+                return None
+            if eos.reward_chain.infused_challenge_chain_sub_slot_hash is not None:
+                return None
 
         self.finished_sub_slots.append((eos, [None] * self.constants.NUM_SPS_SUB_SLOT, total_iters))
 
@@ -671,6 +741,8 @@ def new_peak(
         ip_sub_slot: Optional[EndOfSubSlotBundle],  # None if in first slot
         fork_block: Optional[BlockRecord],
         blocks: BlockchainInterface,
+        next_sub_slot_iters: uint64,
+        next_difficulty: uint64,
     ) -> FullNodeStorePeakResult:
         """
         If the peak is an overflow block, must provide two sub-slots: one for the current sub-slot and one for
@@ -737,7 +809,10 @@ def new_peak(
 
         future_eos: List[EndOfSubSlotBundle] = self.future_eos_cache.get(peak.reward_infusion_new_challenge, []).copy()
         for eos in future_eos:
-            if self.new_finished_sub_slot(eos, blocks, peak, peak_full_block) is not None:
+            if (
+                self.new_finished_sub_slot(eos, blocks, peak, next_sub_slot_iters, next_difficulty, peak_full_block)
+                is not None
+            ):
                 new_eos = eos
                 break
 

chia/timelord/timelord.py

@@ -136,6 +136,7 @@ def __init__(self, root_path: Path, config: Dict[str, Any], constants: Consensus
             self.bluebox_mode = self.config.get("sanitizer_mode", False)
         self.pending_bluebox_info: List[Tuple[float, timelord_protocol.RequestCompactProofOfTime]] = []
         self.last_active_time = time.time()
+        self.max_allowed_inactivity_time = 60
         self.bluebox_pool: Optional[ProcessPoolExecutor] = None
 
     async def _start(self) -> None:
@@ -498,7 +499,7 @@ async def _check_for_new_sp(self, iter_to_look_for: uint64) -> None:
                             self.iters_to_submit[chain].append(next_sp)
                     self.iteration_to_proof_type[next_sp] = IterationType.SIGNAGE_POINT
                     next_iters_count += 1
-                    if next_iters_count == 3:
+                    if next_iters_count == 10:
                         break
 
                 # Break so we alternate between checking SP and IP
@@ -854,9 +855,10 @@ async def _handle_failures(self) -> None:
         else:
             # If there were no failures recently trigger a reset after 60 seconds of no activity.
             # Signage points should be every 9 seconds
-            active_time_threshold = 60
+            active_time_threshold = self.max_allowed_inactivity_time
         if time.time() - self.last_active_time > active_time_threshold:
             log.error(f"Not active for {active_time_threshold} seconds, restarting all chains")
+            self.max_allowed_inactivity_time = min(self.max_allowed_inactivity_time * 2, 1800)
             await self._reset_chains()
 
     async def _manage_chains(self) -> None:

chia/timelord/timelord_api.py

@@ -31,6 +31,7 @@ async def new_peak_timelord(self, new_peak: timelord_protocol.NewPeakTimelord) -
         async with self.timelord.lock:
             if self.timelord.bluebox_mode:
                 return None
+            self.timelord.max_allowed_inactivity_time = 60
             if new_peak.reward_chain_block.weight > self.timelord.last_state.get_weight():
                 log.info("Not skipping peak, don't have. Maybe we are not the fastest timelord")
                 log.info(