Better timestamps on simulated chains (#16326)

.github/workflows/benchmarks.yml

@@ -34,7 +34,7 @@ jobs:
         python-version: [ 3.9 ]
     env:
       CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet
-      BLOCKS_AND_PLOTS_VERSION: 0.32.0
+      BLOCKS_AND_PLOTS_VERSION: 0.33.0
 
     steps:
       - name: Clean workspace

.github/workflows/test-single.yml

@@ -113,7 +113,7 @@ jobs:
       CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet
       CHIA_SIMULATOR_ROOT: ${{ github.workspace }}/.chia/simulator
       JOB_FILE_NAME: tests_${{ matrix.os.file_name }}_python-${{ matrix.python.file_name }}_${{ matrix.configuration.name }}
-      BLOCKS_AND_PLOTS_VERSION: 0.32.0
+      BLOCKS_AND_PLOTS_VERSION: 0.33.0
 
     steps:
       - name: Configure git

chia/simulator/block_tools.py

@@ -4,7 +4,6 @@
 import copy
 import dataclasses
 import logging
-import math
 import os
 import random
 import shutil
@@ -207,7 +206,6 @@ def __init__(
         self.root_path = root_path
         self.log = log
         self.local_keychain = keychain
-        self._block_time_residual = 0.0
         self.local_sk_cache: Dict[bytes32, Tuple[PrivateKey, Any]] = {}
         self.automated_testing = automated_testing
         self.plot_dir_name = plot_dir
@@ -575,7 +573,6 @@ def get_consecutive_blocks(
         previous_generator: Optional[Union[CompressorArg, List[uint32]]] = None,
         genesis_timestamp: Optional[uint64] = None,
         force_plot_id: Optional[bytes32] = None,
-        use_timestamp_residual: bool = False,
     ) -> List[FullBlock]:
         assert num_blocks > 0
         if block_list_input is not None:
@@ -625,7 +622,8 @@ def get_consecutive_blocks(
         curr = latest_block
         while not curr.is_transaction_block:
             curr = blocks[curr.prev_hash]
-        last_timestamp = curr.timestamp
+        assert curr.timestamp is not None
+        last_timestamp = float(curr.timestamp)
         start_height = curr.height
 
         curr = latest_block
@@ -754,10 +752,11 @@ def get_consecutive_blocks(
                             block_generator = None
                             aggregate_signature = G2Element()
 
-                        if not use_timestamp_residual:
-                            self._block_time_residual = 0.0
-
-                        full_block, block_record, self._block_time_residual = get_full_block_and_block_record(
+                        (
+                            full_block,
+                            block_record,
+                            new_timestamp,
+                        ) = get_full_block_and_block_record(
                             constants,
                             blocks,
                             sub_slot_start_total_iters,
@@ -786,17 +785,18 @@ def get_consecutive_blocks(
                             seed,
                             normalized_to_identity_cc_ip=normalized_to_identity_cc_ip,
                             current_time=current_time,
-                            block_time_residual=self._block_time_residual,
                         )
                         if block_record.is_transaction_block:
                             transaction_data_included = True
                             previous_generator = None
                             keep_going_until_tx_block = False
                             assert full_block.foliage_transaction_block is not None
-                            last_timestamp = full_block.foliage_transaction_block.timestamp
-                        else:
-                            if guarantee_transaction_block:
-                                continue
+                        elif guarantee_transaction_block:
+                            continue
+                        # print(f"{full_block.height}: difficulty {difficulty} "
+                        #     f"time: {new_timestamp - last_timestamp:0.2f} "
+                        #     f"tx: {block_record.is_transaction_block}")
+                        last_timestamp = new_timestamp
                         block_list.append(full_block)
                         if full_block.transactions_generator is not None:
                             compressor_arg = detect_potential_template_generator(
@@ -1040,10 +1040,11 @@ def get_consecutive_blocks(
                             block_generator = None
                             aggregate_signature = G2Element()
 
-                        if not use_timestamp_residual:
-                            self._block_time_residual = 0.0
-
-                        full_block, block_record, self._block_time_residual = get_full_block_and_block_record(
+                        (
+                            full_block,
+                            block_record,
+                            new_timestamp,
+                        ) = get_full_block_and_block_record(
                             constants,
                             blocks,
                             sub_slot_start_total_iters,
@@ -1074,17 +1075,19 @@ def get_consecutive_blocks(
                             overflow_rc_challenge=overflow_rc_challenge,
                             normalized_to_identity_cc_ip=normalized_to_identity_cc_ip,
                             current_time=current_time,
-                            block_time_residual=self._block_time_residual,
                         )
 
                         if block_record.is_transaction_block:
                             transaction_data_included = True
                             previous_generator = None
                             keep_going_until_tx_block = False
                             assert full_block.foliage_transaction_block is not None
-                            last_timestamp = full_block.foliage_transaction_block.timestamp
                         elif guarantee_transaction_block:
                             continue
+                        # print(f"{full_block.height}: difficulty {difficulty} "
+                        #     f"time: {new_timestamp - last_timestamp:0.2f} "
+                        #     f"tx: {block_record.is_transaction_block}")
+                        last_timestamp = new_timestamp
 
                         block_list.append(full_block)
                         if full_block.transactions_generator is not None:
@@ -1658,11 +1661,6 @@ def get_icc(
     )
 
 
-def round_timestamp(timestamp: float, residual: float) -> Tuple[int, float]:
-    mod = math.modf(timestamp + residual)
-    return (int(mod[1]), mod[0])
-
-
 def get_full_block_and_block_record(
     constants: ConsensusConstants,
     blocks: Dict[bytes32, BlockRecord],
@@ -1673,7 +1671,7 @@ def get_full_block_and_block_record(
     slot_rc_challenge: bytes32,
     farmer_reward_puzzle_hash: bytes32,
     pool_target: PoolTarget,
-    last_timestamp: uint64,
+    last_timestamp: float,
     start_height: uint32,
     time_per_block: float,
     block_generator: Optional[BlockGenerator],
@@ -1695,13 +1693,16 @@ def get_full_block_and_block_record(
     overflow_rc_challenge: Optional[bytes32] = None,
     normalized_to_identity_cc_ip: bool = False,
     current_time: bool = False,
-    block_time_residual: float = 0.0,
 ) -> Tuple[FullBlock, BlockRecord, float]:
-    time_delta, block_time_residual = round_timestamp(time_per_block, block_time_residual)
+    # we're simulating time between blocks here. The more VDF iterations the
+    # blocks advances, the longer it should have taken (and vice versa). This
+    # formula is meant to converge at 1024 iters per the specified
+    # time_per_block (which defaults to 18.75 seconds)
+    time_per_block *= (((sub_slot_iters / 1024) - 1) * 0.2) + 1
     if current_time is True:
-        timestamp = uint64(max(int(time.time()), last_timestamp + time_delta))
+        timestamp = max(int(time.time()), last_timestamp + time_per_block)
     else:
-        timestamp = uint64(last_timestamp + time_delta)
+        timestamp = last_timestamp + time_per_block
     sp_iters = calculate_sp_iters(constants, sub_slot_iters, signage_point_index)
     ip_iters = calculate_ip_iters(constants, sub_slot_iters, signage_point_index, required_iters)
 
@@ -1719,7 +1720,7 @@ def get_full_block_and_block_record(
         get_plot_signature,
         get_pool_signature,
         signage_point,
-        timestamp,
+        uint64(timestamp),
         BlockCache(blocks),
         seed,
         block_generator,
@@ -1752,7 +1753,7 @@ def get_full_block_and_block_record(
         normalized_to_identity_cc_ip,
     )
 
-    return full_block, block_record, block_time_residual
+    return full_block, block_record, timestamp
 
 
 # these are the costs of unknown conditions, as defined chia_rs here:

tests/farmer_harvester/test_filter_prefix_bits.py

@@ -32,7 +32,7 @@
 # this test
 @pytest.mark.limit_consensus_modes(allowed=[ConsensusMode.PLAIN])
 @pytest.mark.parametrize(
-    argnames=["filter_prefix_bits", "should_pass"], argvalues=[(9, 33), (8, 66), (7, 138), (6, 265), (5, 607)]
+    argnames=["filter_prefix_bits", "should_pass"], argvalues=[(9, 34), (8, 89), (7, 162), (6, 295), (5, 579)]
 )
 def test_filter_prefix_bits_on_blocks(
     default_10000_blocks: List[FullBlock], filter_prefix_bits: uint8, should_pass: int

tools/generate_chain.py

@@ -105,7 +105,6 @@ def main(length: int, fill_rate: int, profile: bool, block_refs: bool, output: O
                 pool_reward_puzzle_hash=pool_puzzlehash,
                 keep_going_until_tx_block=True,
                 genesis_timestamp=uint64(1234567890),
-                use_timestamp_residual=True,
             )
 
             unspent_coins: List[Coin] = []
@@ -164,7 +163,6 @@ def main(length: int, fill_rate: int, profile: bool, block_refs: bool, output: O
                         keep_going_until_tx_block=True,
                         transaction_data=SpendBundle.aggregate(spend_bundles),
                         previous_generator=block_references,
-                        use_timestamp_residual=True,
                     )
                     prev_tx_block = b
                     prev_block = blocks[-2]