diff --git a/.gitignore b/.gitignore index bcd96f8858..ecbeb9a3c3 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,13 @@ tests/core/pyspec/test-reports tests/core/pyspec/eth2spec/test_results.xml *.egg-info + +# flake8 config +tox.ini + +# VS code files +.vscode +*.code-workspace + +# npm (for doctoc) +package-lock.json \ No newline at end of file diff --git a/Makefile b/Makefile index e53aaf8a2a..da87f79126 100644 --- a/Makefile +++ b/Makefile @@ -71,19 +71,19 @@ pyspec: # installs the packages to run pyspec tests install_test: - python3 -m venv venv; . venv/bin/activate; pip3 install .[test] .[lint] + python3.8 -m venv venv; . venv/bin/activate; pip3 install .[lint]; pip3 install -e .[test] test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec find_test: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -k=$(K) --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec + python -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --junitxml=eth2spec/test_results.xml eth2spec + python -m pytest -n 4 --disable-bls --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & diff --git a/configs/README.md b/configs/README.md index 4ca54e0142..be3c60e6f2 100644 --- a/configs/README.md +++ b/configs/README.md @@ -32,4 +32,4 @@ Each preset is a key-value mapping. Presets may contain comments to describe the values. -See [`mainnet.yaml`](./mainnet.yaml) for a complete example. +See [`mainnet_phase0.yaml`](./mainnet_phase0.yaml) for a complete example. diff --git a/configs/mainnet.yaml b/configs/mainnet/phase0.yaml similarity index 68% rename from configs/mainnet.yaml rename to configs/mainnet/phase0.yaml index 42845c235a..39cfddf770 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet/phase0.yaml @@ -76,8 +76,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- -# 86400 seconds (1 day) -MIN_GENESIS_DELAY: 86400 +# 172800 seconds (2 days) +GENESIS_DELAY: 172800 # 12 seconds SECONDS_PER_SLOT: 12 # 2**0 (= 1) slots 12 seconds @@ -151,70 +151,3 @@ DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_SELECTION_PROOF: 0x05000000 DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 -# Phase 1 -DOMAIN_SHARD_PROPOSAL: 0x80000000 -DOMAIN_SHARD_COMMITTEE: 0x81000000 -DOMAIN_LIGHT_CLIENT: 0x82000000 -DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 - - -# Phase 1: Upgrade from Phase 0 -# --------------------------------------------------------------- -PHASE_1_FORK_VERSION: 0x01000000 -# [STUB] -PHASE_1_GENESIS_SLOT: 32 -INITIAL_ACTIVE_SHARDS: 64 - -# Phase 1: General -# --------------------------------------------------------------- -# 2**10` (= 1024) -MAX_SHARDS: 1024 -# 2**3 (= 8) | online epochs | ~51 min -ONLINE_PERIOD: 8 -# 2**7 (= 128) -LIGHT_CLIENT_COMMITTEE_SIZE: 128 -# 2**8 (= 256) | epochs | ~27 hours -LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**18 (= 262,144) -SHARD_BLOCK_CHUNK_SIZE: 262144 -# 2**2 (= 4) -MAX_SHARD_BLOCK_CHUNKS: 4 -# 3 * 2**16` (= 196,608) -TARGET_SHARD_BLOCK_SIZE: 196608 -# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] -# len(SHARD_BLOCK_OFFSETS) -MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 -# 2**14 (= 16,384) Gwei -MAX_GASPRICE: 16384 -# 2**3 (= 8) Gwei -MIN_GASPRICE: 8 -# 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 - - -# Phase 1: Custody Game -# --------------------------------------------------------------- - -# Time parameters -# 2**1 (= 2) epochs, 12.8 minutes -RANDAO_PENALTY_EPOCHS: 2 -# 2**14 (= 16,384) epochs ~73 days -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 16384 -# 2**11 (= 2,048) epochs, ~9 days -EPOCHS_PER_CUSTODY_PERIOD: 2048 -# 2**11 (= 2,048) epochs, ~9 days -CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**7 (= 128) epochs, ~14 hours -MAX_REVEAL_LATENESS_DECREMENT: 128 - -# Max operations -# 2**8 (= 256) -MAX_CUSTODY_KEY_REVEALS: 256 -MAX_EARLY_DERIVED_SECRET_REVEALS: 1 -MAX_CUSTODY_SLASHINGS: 1 - -# Reward and penalty quotients -EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 -# 2**8 (= 256) -MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/mainnet/phase1.yaml b/configs/mainnet/phase1.yaml new file mode 100644 index 0000000000..8c08009576 --- /dev/null +++ b/configs/mainnet/phase1.yaml @@ -0,0 +1,102 @@ +# Mainnet preset - phase 1 + + +# phase1-fork +# --------------------------------------------------------------- +PHASE_1_FORK_VERSION: 0x01000000 +# [STUB] +PHASE_1_GENESIS_SLOT: 32 +INITIAL_ACTIVE_SHARDS: 64 + + +# beacon-chain +# --------------------------------------------------------------- +# Misc +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + +# Shard block configs +# 2**20 (= 1048,576) bytes +MAX_SHARD_BLOCK_SIZE: 1048576 +# 2**18 (= 262,144) bytes +TARGET_SHARD_BLOCK_SIZE: 262144 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**12 (= 4,096) +BYTES_PER_CUSTODY_CHUNK: 4096 +# ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) +CUSTODY_RESPONSE_DEPTH: 8 + +# Gwei values +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 + +# Time parameters +# 2**3 (= 8) | online epochs +ONLINE_PERIOD: 8 +# 2**8 (= 256) | epochs +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 + +# Max operations per block +# 2**20 (= 1,048,576) +MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS: 1048576 + +# Domain types +DOMAIN_SHARD_PROPOSAL: 0x80000000 +DOMAIN_SHARD_COMMITTEE: 0x81000000 +DOMAIN_LIGHT_CLIENT: 0x82000000 +# custody-game spec +DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 +DOMAIN_LIGHT_SELECTION_PROOF: 0x84000000 +DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000 + +# custody-game +# --------------------------------------------------------------- +# Time parameters +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**14 (= 16,384) epochs +CUSTODY_RESPONSE_DEADLINE: 16384 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + +# Misc parameters +# 2**256 - 189 +CUSTODY_PRIME: 115792089237316195423570985008687907853269984665640564039457584007913129639747 +# 3 +CUSTODY_SECRETS: 3 +# 2**5 (= 32) bytes +BYTES_PER_CUSTODY_ATOM: 32 +# 1/1024 chance of custody bit 1 +CUSTODY_PROBABILITY_EXPONENT: 10 + +# Max operations +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/minimal.yaml b/configs/minimal/phase0.yaml similarity index 68% rename from configs/minimal.yaml rename to configs/minimal/phase0.yaml index d8e346ffab..524d0d5f9c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal/phase0.yaml @@ -77,7 +77,7 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- # [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis -MIN_GENESIS_DELAY: 300 +GENESIS_DELAY: 300 # [customized] Faster for testing purposes SECONDS_PER_SLOT: 6 # 2**0 (= 1) slots 6 seconds @@ -151,73 +151,3 @@ DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_SELECTION_PROOF: 0x05000000 DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 -# Phase 1 -DOMAIN_SHARD_PROPOSAL: 0x80000000 -DOMAIN_SHARD_COMMITTEE: 0x81000000 -DOMAIN_LIGHT_CLIENT: 0x82000000 -DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 - - -# Phase 1: Upgrade from Phase 0 -# --------------------------------------------------------------- -# [customized] for testnet distinction -PHASE_1_FORK_VERSION: 0x01000001 -# [customized] for testing -PHASE_1_GENESIS_SLOT: 8 -# [customized] reduced for testing -INITIAL_ACTIVE_SHARDS: 4 - - -# Phase 1: General -# --------------------------------------------------------------- -# [customized] reduced for testing -MAX_SHARDS: 8 -# 2**3 (= 8) | online epochs -ONLINE_PERIOD: 8 -# 2**7 (= 128) -LIGHT_CLIENT_COMMITTEE_SIZE: 128 -# 2**8 (= 256) | epochs -LIGHT_CLIENT_COMMITTEE_PERIOD: 256 -# 2**18 (= 262,144) -SHARD_BLOCK_CHUNK_SIZE: 262144 -# 2**2 (= 4) -MAX_SHARD_BLOCK_CHUNKS: 4 -# 3 * 2**16` (= 196,608) -TARGET_SHARD_BLOCK_SIZE: 196608 -# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. -SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] -# len(SHARD_BLOCK_OFFSETS) -MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 -# 2**14 (= 16,384) Gwei -MAX_GASPRICE: 16384 -# 2**3 (= 8) Gwei -MIN_GASPRICE: 8 -# 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 - - -# Phase 1: Custody Game -# --------------------------------------------------------------- - -# Time parameters -# 2**1 (= 2) epochs -RANDAO_PENALTY_EPOCHS: 2 -# [customized] quicker for testing -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 -# 2**11 (= 2,048) epochs -EPOCHS_PER_CUSTODY_PERIOD: 2048 -# 2**11 (= 2,048) epochs -CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**7 (= 128) epochs -MAX_REVEAL_LATENESS_DECREMENT: 128 - -# Max operations -# 2**8 (= 256) -MAX_CUSTODY_KEY_REVEALS: 256 -MAX_EARLY_DERIVED_SECRET_REVEALS: 1 -MAX_CUSTODY_SLASHINGS: 1 - -# Reward and penalty quotients -EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 -# 2**8 (= 256) -MINOR_REWARD_QUOTIENT: 256 diff --git a/configs/minimal/phase1.yaml b/configs/minimal/phase1.yaml new file mode 100644 index 0000000000..7fbc7e5a30 --- /dev/null +++ b/configs/minimal/phase1.yaml @@ -0,0 +1,106 @@ +# Minimal preset - phase 1 + + +# phase1-fork +# --------------------------------------------------------------- +# [customized] for testnet distinction +PHASE_1_FORK_VERSION: 0x01000001 +# [customized] for testing +PHASE_1_GENESIS_SLOT: 8 +# [customized] reduced for testing +INITIAL_ACTIVE_SHARDS: 2 + + +# beacon-chain +# --------------------------------------------------------------- +# Misc +# [customized] reduced for testing +MAX_SHARDS: 8 +# 2**7 (= 128) +LIGHT_CLIENT_COMMITTEE_SIZE: 128 +# 2**3 (= 8) +GASPRICE_ADJUSTMENT_COEFFICIENT: 8 + +# Shard block configs +# 2**20 (= 1048,576) bytes +MAX_SHARD_BLOCK_SIZE: 1048576 +# 2**18 (= 262,144) bytes +TARGET_SHARD_BLOCK_SIZE: 262144 +# Note: MAX_SHARD_BLOCKS_PER_ATTESTATION is derived from the list length. +SHARD_BLOCK_OFFSETS: [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233] +# len(SHARD_BLOCK_OFFSETS) +MAX_SHARD_BLOCKS_PER_ATTESTATION: 12 +# 2**12 (= 4,096) +BYTES_PER_CUSTODY_CHUNK: 4096 +# ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK) +CUSTODY_RESPONSE_DEPTH: 8 + +# Gwei values +# 2**14 (= 16,384) Gwei +MAX_GASPRICE: 16384 +# 2**3 (= 8) Gwei +MIN_GASPRICE: 8 + +# Time parameters +# 2**3 (= 8) | online epochs +ONLINE_PERIOD: 8 +# 2**8 (= 256) | epochs +LIGHT_CLIENT_COMMITTEE_PERIOD: 256 + +# Max operations per block +# 2**20 (= 1,048,576) +MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS: 1048576 + +# Domain types +DOMAIN_SHARD_PROPOSAL: 0x80000000 +DOMAIN_SHARD_COMMITTEE: 0x81000000 +DOMAIN_LIGHT_CLIENT: 0x82000000 +# custody-game spec +DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000 +DOMAIN_LIGHT_SELECTION_PROOF: 0x84000000 +DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000 + +# custody-game +# --------------------------------------------------------------- +# Time parameters +# 2**1 (= 2) epochs +RANDAO_PENALTY_EPOCHS: 2 +# [customized] quicker for testing +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 128 +# [customized] quicker for testing +EPOCHS_PER_CUSTODY_PERIOD: 64 +# [customized] quicker for testing +CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 +# [customized] quicker for testing +CUSTODY_RESPONSE_DEADLINE: 128 +# [customize for faster testing] +MAX_CHUNK_CHALLENGE_DELAY: 128 + + +# Misc parameters +# 2**256 - 189 +CUSTODY_PRIME: 115792089237316195423570985008687907853269984665640564039457584007913129639747 +# 3 +CUSTODY_SECRETS: 3 +# 2**5 (= 32) bytes +BYTES_PER_CUSTODY_ATOM: 32 +# 1/4 chance of custody bit 1 [customized for faster testing] +CUSTODY_PROBABILITY_EXPONENT: 2 + + +# Max operations +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# [customized] +MAX_CUSTODY_CHUNK_CHALLENGES: 2 +# [customized] +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 8 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 diff --git a/setup.py b/setup.py index 5f0dce7631..53f352b16e 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ def get_spec(file_name: str) -> SpecObject: from lru import LRU -from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint64, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, @@ -118,10 +118,10 @@ def get_spec(file_name: str) -> SpecObject: from lru import LRU -from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy from eth2spec.utils.ssz.ssz_typing import ( View, boolean, Container, List, Vector, uint64, uint8, bit, - ByteList, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, + ByteList, ByteVector, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist, Bitvector, ) from eth2spec.utils import bls @@ -140,7 +140,7 @@ def get_spec(file_name: str) -> SpecObject: def ceillog2(x: uint64) -> int: return (x - 1).bit_length() ''' -SUNDRY_FUNCTIONS = ''' +PHASE0_SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Bytes32] = {} @@ -150,7 +150,10 @@ def get_eth1_data(block: Eth1Block) -> Eth1Data: """ A stub function return mocking Eth1Data. """ - return Eth1Data(block_hash=hash_tree_root(block)) + return Eth1Data( + deposit_root=block.deposit_root, + deposit_count=block.deposit_count, + block_hash=hash_tree_root(block)) def hash(x: bytes) -> Bytes32: # type: ignore @@ -217,6 +220,13 @@ def wrapper(*args, **kw): # type: ignore _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' +PHASE1_SUNDRY_FUNCTIONS = ''' +_get_start_shard = get_start_shard +get_start_shard = cache_this( + lambda state, slot: (state.validators.hash_tree_root(), slot), + _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -247,9 +257,11 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: + '\n\n' + CONFIG_LOADER + '\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec - + '\n' + SUNDRY_FUNCTIONS - + '\n' + + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': + spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + spec += '\n' return spec @@ -382,6 +394,8 @@ def finalize_options(self): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md + specs/phase1/validator.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) @@ -482,6 +496,7 @@ def run(self): url="https://github.com/ethereum/eth2.0-specs", include_package_data=False, package_data={'configs': ['*.yaml'], + 'specs': ['**/*.md'], 'eth2spec': ['VERSION.txt']}, package_dir={ @@ -502,8 +517,9 @@ def run(self): "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.9.4", "py_ecc==4.0.0", + "milagro_bls_binding==1.3.0", "dataclasses==0.6", - "remerkleable==0.1.13", + "remerkleable==0.1.16", "ruamel.yaml==0.16.5", "lru-dict==1.1.6" ] diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 8e212ba13c..74c09a0c57 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -183,6 +183,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | +| `ETH1_FOLLOW_DISTANCE` | `uint64(2**10)` (= 1,024) | | `MAX_COMMITTEES_PER_SLOT` | `uint64(2**6)` (= 64) | | `TARGET_COMMITTEE_SIZE` | `uint64(2**7)` (= 128) | | `MAX_VALIDATORS_PER_COMMITTEE` | `uint64(2**11)` (= 2,048) | @@ -217,6 +218,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | +<<<<<<< HEAD | `MIN_GENESIS_DELAY` | `uint64(86400)` | seconds | 1 day | | `SECONDS_PER_SLOT` | `uint64(12)` | seconds | 12 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `uint64(2**0)` (= 1) | slots | 12 seconds | @@ -228,6 +230,20 @@ The following values are (non-configurable) constants used throughout the specif | `SLOTS_PER_HISTORICAL_ROOT` | `uint64(2**13)` (= 8,192) | slots | ~27 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `uint64(2**8)` (= 256) | epochs | ~27 hours | | `SHARD_COMMITTEE_PERIOD` | `uint64(2**8)` (= 256) | epochs | ~27 hours | +======= +| `GENESIS_DELAY` | `172800` | seconds | 2 days | +| `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | +| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | 14 seconds | +| `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | +| `SLOTS_PER_EPOCH` | `2**5` (= 32) | slots | 6.4 minutes | +| `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | +| `MAX_SEED_LOOKAHEAD` | `2**2` (= 4) | epochs | 25.6 minutes | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | +| `EPOCHS_PER_ETH1_VOTING_PERIOD` | `2**5` (= 32) | epochs | ~3.4 hours | +| `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~27 hours | +| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | +| `SHARD_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | +>>>>>>> dev ### State list lengths @@ -1139,6 +1155,8 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b - `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` - `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` +Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. + ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, eth1_timestamp: uint64, @@ -1149,7 +1167,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, epoch=GENESIS_EPOCH, ) state = BeaconState( - genesis_time=eth1_timestamp - eth1_timestamp % MIN_GENESIS_DELAY + 2 * MIN_GENESIS_DELAY, + genesis_time=eth1_timestamp + GENESIS_DELAY, fork=fork, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), @@ -1357,6 +1375,25 @@ def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) ``` + +```python +def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: + return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) +``` + + +```python +def get_finality_delay(state: BeaconState) -> uint64: + return get_previous_epoch(state) - state.finalized_checkpoint.epoch +``` + + +```python +def is_in_inactivity_leak(state: BeaconState) -> bool: + return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +``` + + ```python def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: previous_epoch = get_previous_epoch(state) @@ -1381,8 +1418,13 @@ def get_attestation_component_deltas(state: BeaconState, for index in get_eligible_validator_indices(state): if index in unslashed_attesting_indices: increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) - rewards[index] += reward_numerator // (total_balance // increment) + if is_in_inactivity_leak(state): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. + rewards[index] += get_base_reward(state, index) + else: + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) else: penalties[index] += get_base_reward(state, index) return rewards, penalties @@ -1429,9 +1471,8 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ a for a in matching_source_attestations if index in get_attesting_indices(state, a.data, a.aggregation_bits) ], key=lambda a: a.inclusion_delay) - proposer_reward = Gwei(get_base_reward(state, index) // PROPOSER_REWARD_QUOTIENT) - rewards[attestation.proposer_index] += proposer_reward - max_attester_reward = get_base_reward(state, index) - proposer_reward + rewards[attestation.proposer_index] += get_proposer_reward(state, index) + max_attester_reward = get_base_reward(state, index) - get_proposer_reward(state, index) rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) # No penalties associated with inclusion delay @@ -1445,16 +1486,16 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S Return inactivity reward/penalty deltas for each validator. """ penalties = [Gwei(0) for _ in range(len(state.validators))] - finality_delay = get_previous_epoch(state) - state.finalized_checkpoint.epoch - - if finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY: + if is_in_inactivity_leak(state): matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) for index in get_eligible_validator_indices(state): - penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * get_base_reward(state, index)) + # If validator is performing optimally this cancels all rewards for a neutral balance + base_reward = get_base_reward(state, index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) if index not in matching_target_attesting_indices: effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT) + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) # No rewards associated with inactivity penalties rewards = [Gwei(0) for _ in range(len(state.validators))] @@ -1725,6 +1766,22 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits +```python +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + ) +``` + ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the Merkle branch @@ -1755,15 +1812,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: return # Add validator and balance entries - state.validators.append(Validator( - pubkey=pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - activation_eligibility_epoch=FAR_FUTURE_EPOCH, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE), - )) + state.validators.append(get_validator_from_deposit(state, deposit)) state.balances.append(amount) else: # Increase balance by deposit amount diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index fb5740ae55..0b14297ad0 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -44,9 +44,11 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0 The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_forkchoice_store(genesis_state)` and update `store` by running: -- `on_tick(time)` whenever `time > store.time` where `time` is the current Unix time -- `on_block(block)` whenever a block `block: SignedBeaconBlock` is received -- `on_attestation(attestation)` whenever an attestation `attestation` is received +- `on_tick(store, time)` whenever `time > store.time` where `time` is the current Unix time +- `on_block(store, block)` whenever a block `block: SignedBeaconBlock` is received +- `on_attestation(store, attestation)` whenever an attestation `attestation` is received + +Any of the above handlers that trigger an unhandled exception (e.g. a failed assert or an out-of-range list access) are considered invalid. Invalid calls to handlers must not modify `store`. *Notes*: @@ -100,7 +102,7 @@ _The block for `anchor_root` is incorrectly initialized to the block header, rat ```python def get_forkchoice_store(anchor_state: BeaconState) -> Store: - anchor_block_header: BeaconBlockHeader = anchor_state.latest_block_header.copy() + anchor_block_header = copy(anchor_state.latest_block_header) if anchor_block_header.state_root == Bytes32(): anchor_block_header.state_root = hash_tree_root(anchor_state) anchor_root = hash_tree_root(anchor_block_header) @@ -114,8 +116,8 @@ def get_forkchoice_store(anchor_state: BeaconState) -> Store: finalized_checkpoint=finalized_checkpoint, best_justified_checkpoint=justified_checkpoint, blocks={anchor_root: anchor_block_header}, - block_states={anchor_root: anchor_state.copy()}, - checkpoint_states={justified_checkpoint: anchor_state.copy()}, + block_states={anchor_root: copy(anchor_state)}, + checkpoint_states={justified_checkpoint: copy(anchor_state)}, ) ``` @@ -150,7 +152,7 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` @@ -285,7 +287,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - # FFG and LMD vote must be consistent with each other + # LMD vote must be consistent with FFG vote target target_slot = compute_start_slot_at_epoch(target.epoch) assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) @@ -300,7 +302,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: # Store target checkpoint state if not yet seen if target not in store.checkpoint_states: - base_state: BeaconState = store.block_states[target.root].copy() + base_state = copy(store.block_states[target.root]) process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) store.checkpoint_states[target] = base_state ``` @@ -342,22 +344,23 @@ def on_tick(store: Store, time: uint64) -> None: ```python def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: block = signed_block.message - # Make a copy of the state to avoid mutability issues + # Parent block must be known assert block.parent_root in store.block_states - pre_state: BeaconState = store.block_states[block.parent_root].copy() + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. assert get_current_slot(store) >= block.slot - # Add new block to the store - store.blocks[hash_tree_root(block)] = block # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, hash_tree_root(block), finalized_slot) == store.finalized_checkpoint.root + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root # Check the block is valid and compute the post-state state = state_transition(pre_state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block # Add new state for this block to the store store.block_states[hash_tree_root(block)] = state @@ -371,15 +374,19 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # Update finalized checkpoint if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: store.finalized_checkpoint = state.finalized_checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - - # Update justified if new justified is later than store justified - # or if store justified is not in chain with finalized checkpoint - if ( - state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch - or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root - ): - store.justified_checkpoint = state.current_justified_checkpoint + + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint ``` #### `on_attestation` diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 0e8699555d..6c4cdd2a68 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -150,6 +150,7 @@ This section outlines constants that are used in this spec. | Name | Value | Description | |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | @@ -203,7 +204,7 @@ Topics are plain UTF-8 strings and are encoded on the wire as determined by prot - `current_fork_version` is the fork version of the epoch of the message to be sent on the topic - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` - `Name` - see table below -- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section for further details. +- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encodings) section for further details. *Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known. @@ -262,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). @@ -274,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -285,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. @@ -391,11 +392,11 @@ The `ErrorMessage` schema is: ``` ( - error_message: String + error_message: List[byte, 256] ) ``` -*Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. +*Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. ### Encoding strategies @@ -443,9 +444,9 @@ In case of an invalid input (header or payload), a reader MUST: All messages that contain only a single field MUST be encoded directly as the type of that field and MUST NOT be encoded as an SSZ container. -Responses that are SSZ-lists (for example `[]SignedBeaconBlock`) send their +Responses that are SSZ-lists (for example `List[SignedBeaconBlock, ...]`) send their constituents individually as `response_chunk`s. For example, the -`[]SignedBeaconBlock` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. +`List[SignedBeaconBlock, ...]` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. ### Messages @@ -468,9 +469,9 @@ The fields are, as seen by the client at the time of sending the message: - `fork_digest`: The node's `ForkDigest` (`compute_fork_digest(current_fork_version, genesis_validators_root)`) where - `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time (not necessarily the epoch to which the node is sync) - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` -- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block. +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. -- `head_root`: The hash_tree_root root of the current head block. +- `head_root`: The `hash_tree_root` root of the current head block (`BeaconBlock`). - `head_slot`: The slot of the block corresponding to the `head_root`. The dialing client MUST send a `Status` request upon connection. @@ -528,11 +529,14 @@ Request Content: Response Content: ``` ( - []SignedBeaconBlock + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] ) ``` -Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. `step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. +`step` defines the slot increment between blocks. For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. +In cases where a slot is empty for a given slot number, no block is returned. For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +A request MUST NOT have a 0 slot increment, i.e. `step >= 1`. `BeaconBlocksByRange` is primarily used to sync historical blocks. @@ -542,15 +546,20 @@ The response MUST consist of zero or more `response_chunk`. Each _successful_ `r Clients MUST keep a record of signed blocks seen since the since the start of the weak subjectivity period and MUST support serving requests of blocks up to their own `head_block_root`. -Clients MUST respond with at least one block, if they have it and it exists in the range. Clients MAY limit the number of blocks in the response. +Clients MUST respond with at least the first block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOCKS` blocks. -The response MUST contain no more than `count` blocks. +The following blocks, where they exist, MUST be send in consecutive order. + +Clients MAY limit the number of blocks in the response. -Clients MUST order blocks by increasing slot number. +The response MUST contain no more than `count` blocks. Clients MUST respond with blocks from their view of the current fork choice -- that is, blocks from the single chain defined by the current head. Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. +This applies to any `step` value. In particular when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. + +After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. #### BeaconBlocksByRoot @@ -560,7 +569,7 @@ Request Content: ``` ( - []Root + List[Root, MAX_REQUEST_BLOCKS] ) ``` @@ -568,12 +577,14 @@ Response Content: ``` ( - []SignedBeaconBlock + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] ) ``` Requests blocks by block root (= `hash_tree_root(SignedBeaconBlock.message)`). The response is a list of `SignedBeaconBlock` whose length is less than or equal to the number of requested blocks. It may be less in the case that the responding peer is missing blocks. +No more than `MAX_REQUEST_BLOCKS` may be requested at a time. + `BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). The request MUST be encoded as an SSZ-field. @@ -1044,7 +1055,7 @@ discv5 uses ENRs and we will presumably need to: Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, clients cannot form valid ENRs prior to this point. ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. Once genesis data is known, we can then form ENRs and safely find peers. -When using an eth1 deposit contract for deposits, `fork_digest` will be known at least `MIN_GENESIS_DELAY` (24 hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. +When using an eth1 deposit contract for deposits, `fork_digest` will be known `GENESIS_DELAY` (48hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. ## Compression/Encoding diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 805498250c..b52d431d75 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -85,11 +85,9 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `ETH1_FOLLOW_DISTANCE` | `2**10` (= 1,024) | blocks | ~4 hours | | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | | `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | | `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | -| `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | ## Becoming a validator @@ -201,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe Specifically a validator should: * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. +* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. @@ -254,11 +252,13 @@ The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 d ###### `Eth1Block` -Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` field available. +Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` and depost contract data available. ```python class Eth1Block(Container): timestamp: uint64 + deposit_root: Root + deposit_count: uint64 # All other eth1 block fields ``` @@ -291,8 +291,14 @@ def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: period_start = voting_period_start_time(state) # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height - votes_to_consider = [get_eth1_data(block) for block in eth1_chain if - is_candidate_block(block, period_start)] + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] # Valid votes already cast during this period valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] @@ -419,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic. ```python -def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: +def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. """ - slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + slots_since_epoch_start = slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start - return uint64((committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT) + return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) ``` ### Attestation aggregation diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 3bb01e2629..4e4778abef 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -12,6 +12,11 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Misc](#misc) + - [Shard block configs](#shard-block-configs) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Time parameters](#time-parameters) + - [Domain types](#domain-types) - [Updated containers](#updated-containers) - [Extended `AttestationData`](#extended-attestationdata) - [Extended `Attestation`](#extended-attestation) @@ -40,20 +45,20 @@ - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) - [`compute_offset_slots`](#compute_offset_slots) - [`compute_updated_gasprice`](#compute_updated_gasprice) + - [`compute_committee_source_epoch`](#compute_committee_source_epoch) - [Beacon state accessors](#beacon-state-accessors) - [`get_active_shard_count`](#get_active_shard_count) - [`get_online_validator_indices`](#get_online_validator_indices) - [`get_shard_committee`](#get_shard_committee) - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - - [`get_indexed_attestation`](#get_indexed_attestation) + - [`get_committee_count_delta`](#get_committee_count_delta) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) - [`get_offset_slots`](#get_offset_slots) - [Predicates](#predicates) - - [Updated `is_valid_indexed_attestation`](#updated-is_valid_indexed_attestation) - - [`is_shard_attestation`](#is_shard_attestation) + - [`is_on_time_attestation`](#is_on_time_attestation) - [`is_winning_attestation`](#is_winning_attestation) - [`optional_aggregate_verify`](#optional_aggregate_verify) - [`optional_fast_aggregate_verify`](#optional_fast_aggregate_verify) @@ -61,14 +66,17 @@ - [Operations](#operations) - [New Attestation processing](#new-attestation-processing) - [`validate_attestation`](#validate_attestation) + - [Updated `process_attestation`](#updated-process_attestation) + - [Shard transition processing](#shard-transition-processing) - [`apply_shard_transition`](#apply_shard_transition) - [`process_crosslink_for_shard`](#process_crosslink_for_shard) - [`process_crosslinks`](#process_crosslinks) - - [`process_attestation`](#process_attestation) - - [New Attester slashing processing](#new-attester-slashing-processing) - - [Shard transition false positives](#shard-transition-false-positives) + - [`verify_empty_shard_transition`](#verify_empty_shard_transition) + - [`process_shard_transitions`](#process_shard_transitions) + - [New default validator for deposits](#new-default-validator-for-deposits) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) + - [Phase 1 final updates](#phase-1-final-updates) - [Custody game updates](#custody-game-updates) - [Online-tracking](#online-tracking) - [Light client committee updates](#light-client-committee-updates) @@ -96,23 +104,53 @@ Configuration is not namespaced. Instead it is strictly an extension; ### Misc -| Name | Value | Unit | Duration | -| - | - | - | - | +| Name | Value | +| - | - | | `MAX_SHARDS` | `2**10` (= 1024) | -| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 min | | `LIGHT_CLIENT_COMMITTEE_SIZE` | `2**7` (= 128) | +| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | + +### Shard block configs + +| Name | Value | Unit | +| - | - | - | +| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | bytes | +| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | bytes | +| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | - | +| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | - | +| `BYTES_PER_CUSTODY_CHUNK` | `2**12` (= 4,096) | bytes | +| `CUSTODY_RESPONSE_DEPTH` | `ceillog2(MAX_SHARD_BLOCK_SIZE // BYTES_PER_CUSTODY_CHUNK)` | - | + +### Gwei values + +| Name | Value | +| - | - | +| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | +| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | + +### Initial values + +| Name | Value | +| - | - | +| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `ONLINE_PERIOD` | `OnlineEpochs(2**3)` (= 8) | online epochs | ~51 mins | | `LIGHT_CLIENT_COMMITTEE_PERIOD` | `Epoch(2**8)` (= 256) | epochs | ~27 hours | -| `MAX_SHARD_BLOCK_SIZE` | `2**20` (= 1,048,576) | | -| `TARGET_SHARD_BLOCK_SIZE` | `2**18` (= 262,144) | | -| `SHARD_BLOCK_OFFSETS` | `[1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233]` | | -| `MAX_SHARD_BLOCKS_PER_ATTESTATION` | `len(SHARD_BLOCK_OFFSETS)` | | -| `MAX_GASPRICE` | `Gwei(2**14)` (= 16,384) | Gwei | | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `2**3` (= 8) | | -| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | | -| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | | -| `NO_SIGNATURE` | `BLSSignature(b'\x00' * 96)` | | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_SHARD_PROPOSAL` | `DomainType('0x80000000')` | +| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +| `DOMAIN_LIGHT_CLIENT` | `DomainType('0x82000000')` | +| `DOMAIN_CUSTODY_BIT_SLASHING` | `DomainType('0x83000000')` | +| `DOMAIN_LIGHT_SELECTION_PROOF` | `DomainType('0x84000000')` | +| `DOMAIN_LIGHT_AGGREGATE_AND_PROOF` | `DomainType('0x85000000')` | ## Updated containers @@ -130,7 +168,7 @@ class AttestationData(Container): source: Checkpoint target: Checkpoint # Current-slot shard block root - head_shard_root: Root + shard_head_root: Root # Shard transition root shard_transition_root: Root ``` @@ -141,7 +179,6 @@ class AttestationData(Container): class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData - custody_bits_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION] signature: BLSSignature ``` @@ -153,6 +190,7 @@ class PendingAttestation(Container): data: AttestationData inclusion_delay: Slot proposer_index: ValidatorIndex + # Phase 1 crosslink_success: boolean ``` @@ -160,8 +198,9 @@ class PendingAttestation(Container): ```python class IndexedAttestation(Container): - committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - attestation: Attestation + attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + signature: BLSSignature ``` ### Extended `AttesterSlashing` @@ -192,7 +231,9 @@ class Validator(Container): # (of the particular validator) in which the validator is activated # = get_custody_period_for_validator(...) next_custody_secret_to_reveal: uint64 - max_reveal_lateness: Epoch + # TODO: The max_reveal_lateness doesn't really make sense anymore. + # So how do we incentivise early custody key reveals now? + all_custody_secrets_revealed_epoch: Epoch # to be initialized to FAR_FUTURE_EPOCH ``` ### Extended `BeaconBlockBody` @@ -211,13 +252,15 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] # Custody game - custody_slashings: List[SignedCustodySlashing, MAX_CUSTODY_SLASHINGS] + chunk_challenges: List[CustodyChunkResponse, MAX_CUSTODY_CHUNK_CHALLENGES] + chunk_challenge_responses: List[CustodyChunkResponse, MAX_CUSTODY_CHUNK_CHALLENGE_RESPONSES] custody_key_reveals: List[CustodyKeyReveal, MAX_CUSTODY_KEY_REVEALS] early_derived_secret_reveals: List[EarlyDerivedSecretReveal, MAX_EARLY_DERIVED_SECRET_REVEALS] + custody_slashings: List[SignedCustodySlashing, MAX_CUSTODY_SLASHINGS] # Shards shard_transitions: Vector[ShardTransition, MAX_SHARDS] # Light clients - light_client_signature_bitfield: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] + light_client_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] light_client_signature: BLSSignature ``` @@ -280,6 +323,7 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Phase 1 + current_epoch_start_shard: Shard shard_states: List[ShardState, MAX_SHARDS] online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT] # not a raw byte array, considered its large size. current_light_committee: CompactCommittee @@ -289,6 +333,8 @@ class BeaconState(Container): # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS exposed_derived_secrets: Vector[List[ValidatorIndex, MAX_EARLY_DERIVED_SECRET_REVEALS * SLOTS_PER_EPOCH], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + custody_chunk_challenge_records: List[CustodyChunkChallengeRecord, MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS] + custody_chunk_challenge_index: uint64 ``` ## New containers @@ -302,6 +348,7 @@ class ShardBlock(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + shard: Shard proposer_index: ValidatorIndex body: ByteList[MAX_SHARD_BLOCK_SIZE] ``` @@ -321,6 +368,7 @@ class ShardBlockHeader(Container): shard_parent_root: Root beacon_parent_root: Root slot: Slot + shard: Shard proposer_index: ValidatorIndex body_root: Root ``` @@ -331,7 +379,6 @@ class ShardBlockHeader(Container): class ShardState(Container): slot: Slot gasprice: Gwei - transition_digest: Bytes32 latest_block_root: Root ``` @@ -344,6 +391,7 @@ class ShardTransition(Container): # Shard block lengths shard_block_lengths: List[uint64, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Shard data roots + # The root is of ByteList[MAX_SHARD_BLOCK_SIZE] shard_data_roots: List[Bytes32, MAX_SHARD_BLOCKS_PER_ATTESTATION] # Intermediate shard states shard_states: List[ShardState, MAX_SHARD_BLOCKS_PER_ATTESTATION] @@ -413,7 +461,7 @@ def unpack_compact_validator(compact_validator: uint64) -> Tuple[ValidatorIndex, ```python def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: """ - Given a state and a list of validator indices, outputs the CompactCommittee representing them. + Given a state and a list of validator indices, outputs the ``CompactCommittee`` representing them. """ validators = [state.validators[i] for i in committee] compact_validators = [ @@ -445,17 +493,30 @@ def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]: #### `compute_updated_gasprice` ```python -def compute_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei: - if length > TARGET_SHARD_BLOCK_SIZE: - delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) +def compute_updated_gasprice(prev_gasprice: Gwei, shard_block_length: uint8) -> Gwei: + if shard_block_length > TARGET_SHARD_BLOCK_SIZE: + delta = (prev_gasprice * (shard_block_length - TARGET_SHARD_BLOCK_SIZE) // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return min(prev_gasprice + delta, MAX_GASPRICE) else: - delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length) + delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - shard_block_length) // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT) return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` +#### `compute_committee_source_epoch` + +```python +def compute_committee_source_epoch(epoch: Epoch, period: uint64) -> Epoch: + """ + Return the source epoch for computing the committee. + """ + source_epoch = epoch - epoch % period + if source_epoch >= period: + source_epoch -= period # `period` epochs lookahead + return source_epoch +``` + ### Beacon state accessors #### `get_active_shard_count` @@ -477,9 +538,10 @@ def get_online_validator_indices(state: BeaconState) -> Set[ValidatorIndex]: ```python def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: - source_epoch = epoch - epoch % SHARD_COMMITTEE_PERIOD - if source_epoch >= SHARD_COMMITTEE_PERIOD: - source_epoch -= SHARD_COMMITTEE_PERIOD + """ + Return the shard committee of the given ``epoch`` of the given ``shard``. + """ + source_epoch = compute_committee_source_epoch(epoch, SHARD_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) active_shard_count = get_active_shard_count(beacon_state) @@ -495,9 +557,10 @@ def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) - ```python def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: - source_epoch = epoch - epoch % LIGHT_CLIENT_COMMITTEE_PERIOD - if source_epoch >= LIGHT_CLIENT_COMMITTEE_PERIOD: - source_epoch -= LIGHT_CLIENT_COMMITTEE_PERIOD + """ + Return the light client committee of no more than ``TARGET_COMMITTEE_SIZE`` validators. + """ + source_epoch = compute_committee_source_epoch(epoch, LIGHT_CLIENT_COMMITTEE_PERIOD) active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) seed = get_seed(beacon_state, source_epoch, DOMAIN_LIGHT_CLIENT) return compute_committee( @@ -512,34 +575,59 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + """ + Return the proposer's index of shard block at ``slot``. + """ + epoch = compute_epoch_at_slot(slot) + committee = get_shard_committee(beacon_state, epoch, shard) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8)) + r = bytes_to_int(seed[:8]) return committee[r % len(committee)] ``` -#### `get_indexed_attestation` +#### `get_committee_count_delta` ```python -def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) -> IndexedAttestation: - committee = get_beacon_committee(beacon_state, attestation.data.slot, attestation.data.index) - return IndexedAttestation( - committee=committee, - attestation=attestation, - ) +def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: + """ + Return the sum of committee counts in range ``[start_slot, stop_slot)``. + """ + return sum(get_committee_count_at_slot(state, Slot(slot)) for slot in range(start_slot, stop_slot)) ``` #### `get_start_shard` ```python def get_start_shard(state: BeaconState, slot: Slot) -> Shard: - # TODO: implement start shard logic - return Shard(0) + """ + Return the start shard at ``slot``. + """ + current_epoch_start_slot = compute_start_slot_at_epoch(get_current_epoch(state)) + active_shard_count = get_active_shard_count(state) + if current_epoch_start_slot == slot: + return state.current_epoch_start_shard + elif slot > current_epoch_start_slot: + # Current epoch or the next epoch lookahead + shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) + else: + # Previous epoch + shard_delta = get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + max_committees_per_epoch = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH + return Shard( + # Ensure positive + (state.current_epoch_start_shard + max_committees_per_epoch * active_shard_count - shard_delta) + % active_shard_count + ) ``` #### `get_shard` ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: + """ + Return the shard that the given ``attestation`` is attesting. + """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -547,6 +635,9 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + """ + Return the latest slot number of the given ``shard``. + """ return state.shard_states[shard].slot ``` @@ -554,68 +645,24 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: - return compute_offset_slots(state.shard_states[shard].slot, state.slot) + """ + Return the offset slots of the given ``shard``. + The offset slot are after the latest slot and before current slot. + """ + return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) ``` ### Predicates -#### Updated `is_valid_indexed_attestation` - -Note that this replaces the Phase 0 `is_valid_indexed_attestation`. +#### `is_on_time_attestation` ```python -def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: +def is_on_time_attestation(state: BeaconState, + attestation: Attestation) -> bool: """ - Check if ``indexed_attestation`` has valid indices and signature. + Check if the given attestation is on-time. """ - # Verify aggregate signature - all_pubkeys = [] - all_signing_roots = [] - attestation = indexed_attestation.attestation - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) - aggregation_bits = attestation.aggregation_bits - if not any(aggregation_bits) or len(aggregation_bits) != len(indexed_attestation.committee): - return False - - if len(attestation.custody_bits_blocks) == 0: - # fall back on phase0 behavior if there is no shard data. - for participant, abit in zip(indexed_attestation.committee, aggregation_bits): - if abit: - all_pubkeys.append(state.validators[participant].pubkey) - signing_root = compute_signing_root(indexed_attestation.attestation.data, domain) - return bls.FastAggregateVerify(all_pubkeys, signing_root, signature=attestation.signature) - else: - for i, custody_bits in enumerate(attestation.custody_bits_blocks): - assert len(custody_bits) == len(indexed_attestation.committee) - for participant, abit, cbit in zip(indexed_attestation.committee, aggregation_bits, custody_bits): - if abit: - all_pubkeys.append(state.validators[participant].pubkey) - # Note: only 2N distinct message hashes - attestation_wrapper = AttestationCustodyBitWrapper( - attestation_data_root=hash_tree_root(attestation.data), - block_index=i, - bit=cbit - ) - all_signing_roots.append(compute_signing_root(attestation_wrapper, domain)) - else: - assert not cbit - return bls.AggregateVerify(all_pubkeys, all_signing_roots, signature=attestation.signature) -``` - -#### `is_shard_attestation` - -```python -def is_shard_attestation(state: BeaconState, - attestation: Attestation, - committee_index: CommitteeIndex) -> bool: - if not ( - attestation.data.index == committee_index - and attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot # Must be on-time attestation - # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 - ): - return False - - return True + return attestation.data.slot == compute_previous_slot(state.slot) ``` #### `is_winning_attestation` @@ -673,9 +720,8 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) process_randao(state, block.body) process_eth1_data(state, block.body) - process_light_client_signatures(state, block.body) + process_light_client_aggregate(state, block.body) process_operations(state, block.body) - verify_shard_transition_false_positives(state, block.body) ``` #### Operations @@ -699,7 +745,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # See custody game spec. process_custody_game_operations(state, body) - process_crosslinks(state, body.shard_transitions, body.attestations) + process_shard_transitions(state, body.shard_transitions, body.attestations) # TODO process_operations(body.shard_receipt_proofs, process_shard_receipt_proofs) ``` @@ -725,20 +771,14 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: else: assert attestation.data.source == state.previous_justified_checkpoint - shard = get_shard(state, attestation) - - # Type 1: on-time attestations, the custody bits should be non-empty. - if attestation.custody_bits_blocks != []: - # Ensure on-time attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot - # Correct data root count - assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) + # Type 1: on-time attestations + if is_on_time_attestation(state, attestation): # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) - # Type 2: no shard transition, no custody bits + # Type 2: no shard transition else: # Ensure delayed attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + assert data.slot < compute_previous_slot(state.slot) # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() @@ -746,6 +786,27 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` +###### Updated `process_attestation` + +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + validate_attestation(state, attestation) + # Store pending attestation for epoch processing + pending_attestation = PendingAttestation( + aggregation_bits=attestation.aggregation_bits, + data=attestation.data, + inclusion_delay=state.slot - attestation.data.slot, + proposer_index=get_beacon_proposer_index(state), + crosslink_success=False, # To be filled in during process_shard_transitions + ) + if attestation.data.target.epoch == get_current_epoch(state): + state.current_epoch_attestations.append(pending_attestation) + else: + state.previous_epoch_attestations.append(pending_attestation) +``` + +##### Shard transition processing + ###### `apply_shard_transition` ```python @@ -767,27 +828,31 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr proposers = [] prev_gasprice = state.shard_states[shard].gasprice shard_parent_root = state.shard_states[shard].latest_block_root - for i in range(len(offset_slots)): + for i, offset_slot in enumerate(offset_slots): shard_block_length = transition.shard_block_lengths[i] shard_state = transition.shard_states[i] # Verify correct calculation of gas prices and slots assert shard_state.gasprice == compute_updated_gasprice(prev_gasprice, shard_block_length) - assert shard_state.slot == offset_slots[i] + assert shard_state.slot == offset_slot # Collect the non-empty proposals result is_empty_proposal = shard_block_length == 0 if not is_empty_proposal: - proposal_index = get_shard_proposer_index(state, offset_slots[i], shard) + proposal_index = get_shard_proposer_index(state, offset_slot, shard) # Reconstruct shard headers header = ShardBlockHeader( shard_parent_root=shard_parent_root, - beacon_parent_root=get_block_root_at_slot(state, offset_slots[i]), + beacon_parent_root=get_block_root_at_slot(state, offset_slot), + slot=offset_slot, + shard=shard, proposer_index=proposal_index, - slot=offset_slots[i], body_root=transition.shard_data_roots[i] ) shard_parent_root = hash_tree_root(header) headers.append(header) proposers.append(proposal_index) + else: + # Must have a stub for `shard_data_root` if empty slot + assert transition.shard_data_roots[i] == Root() prev_gasprice = shard_state.gasprice @@ -801,7 +866,7 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr # Save updated state state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1] - state.shard_states[shard].slot = state.slot - 1 + state.shard_states[shard].slot = compute_previous_slot(state.slot) ``` ###### `process_crosslink_for_shard` @@ -811,9 +876,10 @@ def process_crosslink_for_shard(state: BeaconState, committee_index: CommitteeIndex, shard_transition: ShardTransition, attestations: Sequence[Attestation]) -> Root: - committee = get_beacon_committee(state, state.slot, committee_index) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee = get_beacon_committee(state, on_time_attestation_slot, committee_index) online_indices = get_online_validator_indices(state) - shard = compute_shard_from_committee_index(state, committee_index, state.slot) + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) @@ -823,7 +889,7 @@ def process_crosslink_for_shard(state: BeaconState, for attestation in transition_attestations: participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) transition_participants = transition_participants.union(participants) - assert attestation.data.head_shard_root == shard_transition.shard_data_roots[ + assert attestation.data.shard_head_root == shard_transition.shard_data_roots[ len(shard_transition.shard_data_roots) - 1 ] @@ -868,17 +934,16 @@ def process_crosslink_for_shard(state: BeaconState, def process_crosslinks(state: BeaconState, shard_transitions: Sequence[ShardTransition], attestations: Sequence[Attestation]) -> None: - committee_count = get_committee_count_at_slot(state, state.slot) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee_count = get_committee_count_at_slot(state, on_time_attestation_slot) for committee_index in map(CommitteeIndex, range(committee_count)): - shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot - shard_transition = shard_transitions[shard] shard_attestations = [ attestation for attestation in attestations - if is_shard_attestation(state, attestation, committee_index) + if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] - - winning_root = process_crosslink_for_shard(state, committee_index, shard_transition, shard_attestations) + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) + winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink for pending_attestation in state.current_epoch_attestations: @@ -886,95 +951,77 @@ def process_crosslinks(state: BeaconState, pending_attestation.crosslink_success = True ``` -###### `process_attestation` +###### `verify_empty_shard_transition` ```python -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - validate_attestation(state, attestation) - # Store pending attestation for epoch processing - pending_attestation = PendingAttestation( - aggregation_bits=attestation.aggregation_bits, - data=attestation.data, - inclusion_delay=state.slot - attestation.data.slot, - proposer_index=get_beacon_proposer_index(state), - crosslink_success=False, # To be filled in during process_crosslinks - ) - if attestation.data.target.epoch == get_current_epoch(state): - state.current_epoch_attestations.append(pending_attestation) - else: - state.previous_epoch_attestations.append(pending_attestation) +def verify_empty_shard_transition(state: BeaconState, shard_transitions: Sequence[ShardTransition]) -> bool: + """ + Verify that a `shard_transition` in a block is empty if an attestation was not processed for it. + """ + for shard in range(get_active_shard_count(state)): + if state.shard_states[shard].slot != compute_previous_slot(state.slot): + if shard_transitions[shard] != ShardTransition(): + return False + return True ``` -##### New Attester slashing processing +###### `process_shard_transitions` ```python -def get_indices_from_committee( - committee: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE], - bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE]: - assert len(bits) == len(committee) - return List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE]( - [validator_index for i, validator_index in enumerate(committee) if bits[i]] - ) +def process_shard_transitions(state: BeaconState, + shard_transitions: Sequence[ShardTransition], + attestations: Sequence[Attestation]) -> None: + # Process crosslinks + process_crosslinks(state, shard_transitions, attestations) + # Verify the empty proposal shard states + assert verify_empty_shard_transition(state, shard_transitions) ``` -```python -def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: - indexed_attestation_1 = attester_slashing.attestation_1 - indexed_attestation_2 = attester_slashing.attestation_2 +##### New default validator for deposits - assert is_slashable_attestation_data( - indexed_attestation_1.attestation.data, - indexed_attestation_2.attestation.data, +```python +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + next_custody_secret_to_reveal = get_custody_period_for_validator( + ValidatorIndex(len(state.validators)), + get_current_epoch(state), ) - assert is_valid_indexed_attestation(state, indexed_attestation_1) - assert is_valid_indexed_attestation(state, indexed_attestation_2) - indices_1 = get_indices_from_committee( - indexed_attestation_1.committee, - indexed_attestation_1.attestation.aggregation_bits, - ) - indices_2 = get_indices_from_committee( - indexed_attestation_2.committee, - indexed_attestation_2.attestation.aggregation_bits, + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + next_custody_secret_to_reveal=next_custody_secret_to_reveal, + all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, ) - - slashed_any = False - indices = set(indices_1).intersection(indices_2) - for index in sorted(indices): - if is_slashable_validator(state.validators[index], get_current_epoch(state)): - slash_validator(state, index) - slashed_any = True - assert slashed_any -``` - -#### Shard transition false positives - -```python -def verify_shard_transition_false_positives(state: BeaconState, block_body: BeaconBlockBody) -> None: - # Verify that a `shard_transition` in a block is empty if an attestation was not processed for it - for shard in range(get_active_shard_count(state)): - if state.shard_states[shard].slot != state.slot - 1: - assert block_body.shard_transitions[shard] == ShardTransition() ``` #### Light client processing ```python -def process_light_client_signatures(state: BeaconState, block_body: BeaconBlockBody) -> None: +def process_light_client_aggregate(state: BeaconState, block_body: BeaconBlockBody) -> None: committee = get_light_client_committee(state, get_current_epoch(state)) + previous_slot = compute_previous_slot(state.slot) + previous_block_root = get_block_root_at_slot(state, previous_slot) + total_reward = Gwei(0) signer_pubkeys = [] for bit_index, participant_index in enumerate(committee): - if block_body.light_client_signature_bitfield[bit_index]: + if block_body.light_client_bits[bit_index]: signer_pubkeys.append(state.validators[participant_index].pubkey) - increase_balance(state, participant_index, get_base_reward(state, participant_index)) - total_reward += get_base_reward(state, participant_index) + if not state.validators[participant_index].slashed: + increase_balance(state, participant_index, get_base_reward(state, participant_index)) + total_reward += get_base_reward(state, participant_index) increase_balance(state, get_beacon_proposer_index(state), Gwei(total_reward // PROPOSER_REWARD_QUOTIENT)) - slot = compute_previous_slot(state.slot) - signing_root = compute_signing_root(get_block_root_at_slot(state, slot), - get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(slot))) + signing_root = compute_signing_root(previous_block_root, + get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(previous_slot))) assert optional_fast_aggregate_verify(signer_pubkeys, signing_root, block_body.light_client_signature) ``` @@ -988,16 +1035,27 @@ def process_epoch(state: BeaconState) -> None: process_rewards_and_penalties(state) process_registry_updates(state) process_reveal_deadlines(state) + process_challenge_deadlines(state) process_slashings(state) - process_final_updates(state) + process_final_updates(state) # phase 0 final updates + process_phase_1_final_updates(state) +``` + +#### Phase 1 final updates + +```python +def process_phase_1_final_updates(state: BeaconState) -> None: process_custody_final_updates(state) process_online_tracking(state) process_light_client_committee_updates(state) + + # Update current_epoch_start_shard + state.current_epoch_start_shard = get_start_shard(state, Slot(state.slot + 1)) ``` #### Custody game updates -`process_reveal_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./1_custody-game.md), +`process_reveal_deadlines`, `process_challenge_deadlines` and `process_custody_final_updates` are defined in [the Custody Game spec](./custody-game.md), #### Online-tracking @@ -1018,7 +1076,9 @@ def process_online_tracking(state: BeaconState) -> None: ```python def process_light_client_committee_updates(state: BeaconState) -> None: - # Update light client committees + """ + Update light client committees. + """ if get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == 0: state.current_light_committee = state.next_light_committee new_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 5f5acd84fe..124167a358 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -15,21 +15,28 @@ - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) - - [Signature domain types](#signature-domain-types) - [Data structures](#data-structures) - [New Beacon Chain operations](#new-beacon-chain-operations) + - [`CustodyChunkChallenge`](#custodychunkchallenge) + - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) + - [`CustodyChunkResponse`](#custodychunkresponse) - [`CustodySlashing`](#custodyslashing) - [`SignedCustodySlashing`](#signedcustodyslashing) - [`CustodyKeyReveal`](#custodykeyreveal) - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Helpers](#helpers) + - [`replace_empty_or_append`](#replace_empty_or_append) - [`legendre_bit`](#legendre_bit) - - [`custody_atoms`](#custody_atoms) + - [`get_custody_atoms`](#get_custody_atoms) + - [`get_custody_secrets`](#get_custody_secrets) + - [`universal_hash_function`](#universal_hash_function) - [`compute_custody_bit`](#compute_custody_bit) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_custody_period_for_validator`](#get_custody_period_for_validator) - [Per-block processing](#per-block-processing) - [Custody Game Operations](#custody-game-operations) + - [Chunk challenges](#chunk-challenges) + - [Custody chunk response](#custody-chunk-response) - [Custody key reveals](#custody-key-reveals) - [Early derived secret reveals](#early-derived-secret-reveals) - [Custody Slashings](#custody-slashings) @@ -49,8 +56,10 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | Unit | | - | - | - | -| `BLS12_381_Q` | `4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787` | - | -| `BYTES_PER_CUSTODY_ATOM` | `48` | bytes | +| `CUSTODY_PRIME` | `2 ** 256 - 189` | - | +| `CUSTODY_SECRETS` | `3` | - | +| `BYTES_PER_CUSTODY_ATOM` | `32` | bytes | +| `CUSTODY_PROBABILITY_EXPONENT` | `10` | - | ## Configuration @@ -59,18 +68,22 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `RANDAO_PENALTY_EPOCHS` | `2**1` (= 2) | epochs | 12.8 minutes | -| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` (= 16,384) | epochs | ~73 days | -| `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | +| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**15` (= 32,768) | epochs | ~146 days | +| `EPOCHS_PER_CUSTODY_PERIOD` | `2**14` (= 16,384) | epochs | ~73 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | -| `MAX_REVEAL_LATENESS_DECREMENT` | `2**7` (= 128) | epochs | ~14 hours | +| `MAX_CHUNK_CHALLENGE_DELAY` | `2**15` (= 32,768) | epochs | ~146 days | +| `CHUNK_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | ### Max operations per block | Name | Value | | - | - | +| `MAX_CUSTODY_CHUNK_CHALLENGE_RECORDS` | `2**20` (= 1,048,576) | | `MAX_CUSTODY_KEY_REVEALS` | `2**8` (= 256) | -| `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | -| `MAX_CUSTODY_SLASHINGS` | `1` | +| `MAX_EARLY_DERIVED_SECRET_REVEALS` | `2**0` (= 1) | +| `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | +| `MAX_CUSTODY_CHUNK_CHALLENGE_RESPONSES` | `2**4` (= 16) | +| `MAX_CUSTODY_SLASHINGS` | `2**0` (= 1) | ### Reward and penalty quotients @@ -79,23 +92,47 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | | `MINOR_REWARD_QUOTIENT` | `2**8` (= 256) | -### Signature domain types +## Data structures -The following types are defined, mapping into `DomainType` (little endian): +### New Beacon Chain operations -| Name | Value | -| - | - | -| `DOMAIN_CUSTODY_BIT_SLASHING` | `DomainType('0x83000000')` | +#### `CustodyChunkChallenge` -## Data structures +```python +class CustodyChunkChallenge(Container): + responder_index: ValidatorIndex + shard_transition: ShardTransition + attestation: Attestation + data_index: uint64 + chunk_index: uint64 +``` -### New Beacon Chain operations +#### `CustodyChunkChallengeRecord` + +```python +class CustodyChunkChallengeRecord(Container): + challenge_index: uint64 + challenger_index: ValidatorIndex + responder_index: ValidatorIndex + inclusion_epoch: Epoch + data_root: Root + chunk_index: uint64 +``` + +#### `CustodyChunkResponse` + +```python +class CustodyChunkResponse(Container): + challenge_index: uint64 + chunk_index: uint64 + chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] + branch: Vector[Root, CUSTODY_RESPONSE_DEPTH] +``` #### `CustodySlashing` ```python class CustodySlashing(Container): - # Attestation.custody_bits_blocks[data_index][committee.index(malefactor_index)] is the target custody bit to check. # (Attestation.data.shard_transition_root as ShardTransition).shard_data_roots[data_index] is the root of the data. data_index: uint64 malefactor_index: ValidatorIndex @@ -114,7 +151,6 @@ class SignedCustodySlashing(Container): signature: BLSSignature ``` - #### `CustodyKeyReveal` ```python @@ -146,6 +182,18 @@ class EarlyDerivedSecretReveal(Container): ## Helpers +### `replace_empty_or_append` + +```python +def replace_empty_or_append(l: List, new_element: Any) -> int: + for i in range(len(l)): + if l[i] == type(new_element)(): + l[i] = new_element + return i + l.append(new_element) + return len(l) - 1 +``` + ### `legendre_bit` Returns the Legendre symbol `(a/q)` normalizes as a bit (i.e. `((a/q) + 1) // 2`). In a production implementation, a well-optimized library (e.g. GMP) should be used for this. @@ -175,7 +223,7 @@ def legendre_bit(a: int, q: int) -> int: return 0 ``` -### `custody_atoms` +### `get_custody_atoms` Given one set of data, return the custody atoms: each atom will be combined with one legendre bit. @@ -186,16 +234,42 @@ def get_custody_atoms(bytez: bytes) -> Sequence[bytes]: for i in range(0, len(bytez), BYTES_PER_CUSTODY_ATOM)] ``` -### `compute_custody_bit` +### `get_custody_secrets` + +Extract the custody secrets from the signature ```python -def compute_custody_bit(key: BLSSignature, data: bytes) -> bit: +def get_custody_secrets(key: BLSSignature) -> Sequence[int]: full_G2_element = bls.signature_to_G2(key) - s = full_G2_element[0].coeffs + signature = full_G2_element[0].coeffs + signature_bytes = b"".join(x.to_bytes(48, "little") for x in signature) + secrets = [int.from_bytes(signature_bytes[i:i + BYTES_PER_CUSTODY_ATOM], "little") + for i in range(0, len(signature_bytes), 32)] + return secrets +``` + +### `universal_hash_function` + +```python +def universal_hash_function(data_chunks: Sequence[bytes], secrets: Sequence[int]) -> int: + n = len(data_chunks) + return ( + sum( + secrets[i % CUSTODY_SECRETS]**i * int.from_bytes(atom, "little") % CUSTODY_PRIME + for i, atom in enumerate(data_chunks) + ) + secrets[n % CUSTODY_SECRETS]**n + ) % CUSTODY_PRIME +``` + +### `compute_custody_bit` + +```python +def compute_custody_bit(key: BLSSignature, data: ByteList[MAX_SHARD_BLOCK_SIZE]) -> bit: custody_atoms = get_custody_atoms(data) - n = len(custody_atoms) - a = sum(s[i % 2]**i * int.from_bytes(atom, "little") for i, atom in enumerate(custody_atoms) + s[n % 2]**n) - return legendre_bit(a, BLS12_381_Q) + secrets = get_custody_secrets(key) + uhf = universal_hash_function(custody_atoms, secrets) + legendre_bits = [legendre_bit(uhf + secrets[0] + i, CUSTODY_PRIME) for i in range(CUSTODY_PROBABILITY_EXPONENT)] + return all(legendre_bits) ``` ### `get_randao_epoch_for_custody_period` @@ -227,11 +301,90 @@ def process_custody_game_operations(state: BeaconState, body: BeaconBlockBody) - for operation in operations: fn(state, operation) + for_ops(body.chunk_challenges, process_chunk_challenge) + for_ops(body.chunk_challenge_responses, process_chunk_challenge) for_ops(body.custody_key_reveals, process_custody_key_reveal) for_ops(body.early_derived_secret_reveals, process_early_derived_secret_reveal) for_ops(body.custody_slashings, process_custody_slashing) ``` +#### Chunk challenges + +```python +def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: + # Verify the attestation + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, challenge.attestation)) + # Verify it is not too late to challenge the attestation + max_attestation_challenge_epoch = challenge.attestation.data.target.epoch + MAX_CHUNK_CHALLENGE_DELAY + assert get_current_epoch(state) <= max_attestation_challenge_epoch + # Verify it is not too late to challenge the responder + responder = state.validators[challenge.responder_index] + if responder.exit_epoch < FAR_FUTURE_EPOCH: + assert get_current_epoch(state) <= responder.exit_epoch + MAX_CHUNK_CHALLENGE_DELAY + # Verify responder is slashable + assert is_slashable_validator(responder, get_current_epoch(state)) + # Verify the responder participated in the attestation + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bits) + assert challenge.responder_index in attesters + # Verify shard transition is correctly given + assert hash_tree_root(challenge.shard_transition) == challenge.attestation.data.shard_transition_root + data_root = challenge.shard_transition.shard_data_roots[challenge.data_index] + # Verify the challenge is not a duplicate + for record in state.custody_chunk_challenge_records: + assert ( + record.data_root != data_root or + record.chunk_index != challenge.chunk_index + ) + # Verify depth + shard_block_length = challenge.shard_transition.shard_block_lengths[challenge.data_index] + transition_chunks = (shard_block_length + BYTES_PER_CUSTODY_CHUNK - 1) // BYTES_PER_CUSTODY_CHUNK + assert challenge.chunk_index < transition_chunks + # Add new chunk challenge record + new_record = CustodyChunkChallengeRecord( + challenge_index=state.custody_chunk_challenge_index, + challenger_index=get_beacon_proposer_index(state), + responder_index=challenge.responder_index, + inclusion_epoch=get_current_epoch(state), + data_root=challenge.shard_transition.shard_data_roots[challenge.data_index], + chunk_index=challenge.chunk_index, + ) + replace_empty_or_append(state.custody_chunk_challenge_records, new_record) + + state.custody_chunk_challenge_index += 1 + # Postpone responder withdrawability + responder.withdrawable_epoch = FAR_FUTURE_EPOCH +``` + +#### Custody chunk response + +```python +def process_chunk_challenge_response(state: BeaconState, + response: CustodyChunkResponse) -> None: + # Get matching challenge (if any) from records + matching_challenges = [ + record for record in state.custody_chunk_challenge_records + if record.challenge_index == response.challenge_index + ] + assert len(matching_challenges) == 1 + challenge = matching_challenges[0] + # Verify chunk index + assert response.chunk_index == challenge.chunk_index + # Verify the chunk matches the crosslink data root + assert is_valid_merkle_branch( + leaf=hash_tree_root(response.chunk), + branch=response.branch, + depth=CUSTODY_RESPONSE_DEPTH, + index=response.chunk_index, + root=challenge.data_root, + ) + # Clear the challenge + index_in_records = state.custody_chunk_challenge_records.index(challenge) + state.custody_chunk_challenge_records[index_in_records] = CustodyChunkChallengeRecord() + # Reward the proposer + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) +``` + #### Custody key reveals ```python @@ -244,7 +397,14 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_secret_to_reveal, reveal.revealer_index) custody_reveal_period = get_custody_period_for_validator(reveal.revealer_index, get_current_epoch(state)) - assert revealer.next_custody_secret_to_reveal < custody_reveal_period + # Only past custody periods can be revealed, except after exiting the exit period can be revealed + is_past_reveal = revealer.next_custody_secret_to_reveal < custody_reveal_period + is_exited = revealer.exit_epoch <= get_current_epoch(state) + is_exit_period_reveal = ( + revealer.next_custody_secret_to_reveal + == get_custody_period_for_validator(reveal.revealer_index, revealer.exit_epoch - 1) + ) + assert is_past_reveal or (is_exited and is_exit_period_reveal) # Revealed validator is active or exited, but not withdrawn assert is_slashable_validator(revealer, get_current_epoch(state)) @@ -254,19 +414,9 @@ def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> signing_root = compute_signing_root(epoch_to_sign, domain) assert bls.Verify(revealer.pubkey, signing_root, reveal.reveal) - # Decrement max reveal lateness if response is timely - if epoch_to_sign + EPOCHS_PER_CUSTODY_PERIOD >= get_current_epoch(state): - if revealer.max_reveal_lateness >= MAX_REVEAL_LATENESS_DECREMENT: - revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT - else: - revealer.max_reveal_lateness = 0 - else: - revealer.max_reveal_lateness = max( - revealer.max_reveal_lateness, - get_current_epoch(state) - epoch_to_sign - EPOCHS_PER_CUSTODY_PERIOD - ) - # Process reveal + if is_exited and is_exit_period_reveal: + revealer.all_custody_secrets_revealed_epoch = get_current_epoch(state) revealer.next_custody_secret_to_reveal += 1 # Reward Block Proposer @@ -359,14 +509,16 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed # Verify the attestation assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) - # TODO: custody_slashing.data is not chunked like shard blocks yet, result is lots of padding. - # TODO: can do a single combined merkle proof of data being attested. # Verify the shard transition is indeed attested by the attestation shard_transition = custody_slashing.shard_transition - assert hash_tree_root(shard_transition) == attestation.shard_transition_root + assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root # Verify that the provided data matches the shard-transition - assert hash_tree_root(custody_slashing.data) == shard_transition.shard_data_roots[custody_slashing.data_index] + assert ( + custody_slashing.data.get_backing().get_left().merkle_root() + == shard_transition.shard_data_roots[custody_slashing.data_index] + ) + assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index] # Verify existence and participation of claimed malefactor attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) @@ -381,18 +533,14 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed signing_root = compute_signing_root(epoch_to_sign, domain) assert bls.Verify(malefactor.pubkey, signing_root, custody_slashing.malefactor_secret) - # Get the custody bit - custody_bits = attestation.custody_bits_blocks[custody_slashing.data_index] - committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - claimed_custody_bit = custody_bits[committee.index(custody_slashing.malefactor_index)] - # Compute the custody bit computed_custody_bit = compute_custody_bit(custody_slashing.malefactor_secret, custody_slashing.data) - + # Verify the claim - if claimed_custody_bit != computed_custody_bit: + if computed_custody_bit == 1: # Slash the malefactor, reward the other committee members slash_validator(state, custody_slashing.malefactor_index) + committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) others_count = len(committee) - 1 whistleblower_reward = Gwei(malefactor.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT // others_count) for attester_index in attesters: @@ -409,28 +557,44 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed ### Handling of reveal deadlines -Run `process_reveal_deadlines(state)` after `process_registry_updates(state)`: - ```python def process_reveal_deadlines(state: BeaconState) -> None: epoch = get_current_epoch(state) for index, validator in enumerate(state.validators): - if get_custody_period_for_validator(ValidatorIndex(index), epoch) > validator.next_custody_secret_to_reveal: - # ------------------ WARNING ----------------------- # - # UNSAFE REMOVAL OF SLASHING TO PRIORITIZE PHASE 0 CI # - # Must find generic way to handle key reveals in tests # - # ---------------------------------------------------- # - - # slash_validator(state, ValidatorIndex(index)) - pass + deadline = validator.next_custody_secret_to_reveal + 1 + if get_custody_period_for_validator(ValidatorIndex(index), epoch) > deadline: + slash_validator(state, ValidatorIndex(index)) ``` -### Final updates +```python +def process_challenge_deadlines(state: BeaconState) -> None: + for custody_chunk_challenge in state.custody_chunk_challenge_records: + if get_current_epoch(state) > custody_chunk_challenge.inclusion_epoch + EPOCHS_PER_CUSTODY_PERIOD: + slash_validator(state, custody_chunk_challenge.responder_index, custody_chunk_challenge.challenger_index) + index_in_records = state.custody_chunk_challenge_records.index(custody_chunk_challenge) + state.custody_chunk_challenge_records[index_in_records] = CustodyChunkChallengeRecord() +``` -After `process_final_updates(state)`, additional updates are made for the custody game: +### Final updates ```python def process_custody_final_updates(state: BeaconState) -> None: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[get_current_epoch(state) % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] + + # Reset withdrawable epochs if challenge records are empty + records = state.custody_chunk_challenge_records + validator_indices_in_records = set([record.responder_index for record in records]) + for index, validator in enumerate(state.validators): + if validator.exit_epoch != FAR_FUTURE_EPOCH: + not_all_secrets_are_revealed = validator.all_custody_secrets_revealed_epoch == FAR_FUTURE_EPOCH + if index in validator_indices_in_records or not_all_secrets_are_revealed: + # Delay withdrawable epochs if challenge records are not empty or not all + # custody secrets revealed + validator.withdrawable_epoch = FAR_FUTURE_EPOCH + else: + # Reset withdrawable epochs if challenge records are empty and all secrets are revealed + if validator.withdrawable_epoch == FAR_FUTURE_EPOCH: + validator.withdrawable_epoch = Epoch(validator.all_custody_secrets_revealed_epoch + + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) ``` diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa090..f92640ebb3 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -9,8 +9,9 @@ - [Introduction](#introduction) -- [Fork choice](#fork-choice) - - [Handlers](#handlers) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) @@ -19,34 +20,29 @@ This document is the beacon chain fork choice spec for part of Ethereum 2.0 Phase 1. -## Fork choice +### Helpers -Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_attestation` must be re-specified to handle this. The bulk of `on_attestation` has been moved out into a few helpers to reduce code duplication where possible. +#### Extended `LatestMessage` -The rest of the fork choice remains stable. +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` -### Handlers +#### Updated `update_latest_messages` ```python -def on_attestation(store: Store, attestation: Attestation) -> None: - """ - Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. - - An ``attestation`` that is asserted as invalid may be valid at a later time, - consider scheduling it for later processing in such case. - """ - validate_on_attestation(store, attestation) - store_target_checkpoint_state(store, attestation.data.target) - - # Get state at the `target` to fully validate attestation - target_state = store.checkpoint_states[attestation.data.target] - indexed_attestation = get_indexed_attestation(target_state, attestation) - assert is_valid_indexed_attestation(target_state, indexed_attestation) - - # Update latest messages for attesting indices - attesting_indices = [ - index for i, index in enumerate(indexed_attestation.committee) - if attestation.aggregation_bits[i] - ] - update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root + ) +``` diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index cc7d8f33ee..496191e073 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -80,7 +80,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: exit_epoch=phase0_validator.exit_epoch, withdrawable_epoch=phase0_validator.withdrawable_epoch, next_custody_secret_to_reveal=get_custody_period_for_validator(ValidatorIndex(i), epoch), - max_reveal_lateness=0, # TODO custody refactor. Outdated? + all_custody_secrets_revealed_epoch=FAR_FUTURE_EPOCH, ) for i, phase0_validator in enumerate(pre.validators) ), balances=pre.balances, @@ -99,11 +99,11 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, # Phase 1 + current_epoch_start_shard=Shard(0), shard_states=List[ShardState, MAX_SHARDS]( ShardState( slot=pre.slot, gasprice=MIN_GASPRICE, - transition_digest=Root(), latest_block_root=Root(), ) for i in range(INITIAL_ACTIVE_SHARDS) ), diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 0000000000..411ad9b6b7 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,182 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). + +## Fork choice + +### Helpers + +#### `ShardStore` + +```python +@dataclass +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) +``` + +#### `get_forkchoice_shard_store` + +```python +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: + # Execute the LMD-GHOST fork choice + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } + while True: + # Find the valid child block roots + children = [ + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root + ] + if len(children) == 0: + return shard_head_root + # Sort by latest attesting balance with ties broken lexicographically + shard_head_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] + if block.slot > slot: + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` + +#### `get_pending_shard_blocks` + +```python +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the canonical shard block branch that has not yet been crosslinked. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + shard = shard_store.shard + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists + assert shard_block.shard_parent_root in shard_store.block_states + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] + + # Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] + + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot + + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert ( + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Check the block is valid and compute the post-state + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + + post_state = get_post_shard_state(shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + + # Add new state for this block to the store + shard_store.block_states[hash_tree_root(shard_block)] = post_state +``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 55b867faa9..24c39aa3d2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -10,14 +10,10 @@ - [Introduction](#introduction) - [Helper functions](#helper-functions) - - [Misc](#misc) - [Shard block verification functions](#shard-block-verification-functions) - [Shard state transition](#shard-state-transition) - [Fraud proofs](#fraud-proofs) - [Verifying the proof](#verifying-the-proof) -- [Honest committee member behavior](#honest-committee-member-behavior) - - [Helper functions](#helper-functions-1) - - [Make attestations](#make-attestations) @@ -27,30 +23,30 @@ This document describes the shard transition function and fraud proofs as part o ## Helper functions -### Misc - -```python -def compute_shard_transition_digest(beacon_state: BeaconState, - shard_state: ShardState, - beacon_parent_root: Root, - shard_body_root: Root) -> Bytes32: - # TODO: use SSZ hash tree root - return hash( - hash_tree_root(shard_state) + beacon_parent_root + shard_body_root - ) -``` - ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) +def verify_shard_block_message(beacon_parent_state: BeaconState, + shard_parent_state: ShardState, + block: ShardBlock) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots + # Check `shard` field + assert block.shard == shard + # Check `proposer_index` field + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -67,38 +63,31 @@ def verify_shard_block_signature(beacon_state: BeaconState, ## Shard state transition ```python -def shard_state_transition(beacon_state: BeaconState, - shard_state: ShardState, +def shard_state_transition(shard_state: ShardState, block: ShardBlock) -> None: - # Update shard state + """ + Update ``shard_state`` with shard ``block``. + """ + shard_state.slot = block.slot prev_gasprice = shard_state.gasprice + shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) if len(block.body) == 0: latest_block_root = shard_state.latest_block_root else: latest_block_root = hash_tree_root(block) - - shard_state.transition_digest = compute_shard_transition_digest( - beacon_state, - shard_state, - block.beacon_parent_root, - block.body, - ) - shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) - shard_state.slot = block.slot shard_state.latest_block_root = latest_block_root ``` We have a pure function `get_post_shard_state` for describing the fraud proof verification and honest validator behavior. ```python -def get_post_shard_state(beacon_state: BeaconState, - shard_state: ShardState, +def get_post_shard_state(shard_state: ShardState, block: ShardBlock) -> ShardState: """ A pure function that returns a new post ShardState instead of modifying the given `shard_state`. """ post_state = shard_state.copy() - shard_state_transition(beacon_state, post_state, block) + shard_state_transition(post_state, block) return post_state ``` @@ -140,8 +129,8 @@ def is_valid_fraud_proof(beacon_state: BeaconState, else: shard_state = transition.shard_states[offset_index - 1] # Not doing the actual state updates here. - shard_state = get_post_shard_state(beacon_state, shard_state, block) - if shard_state.transition_digest != transition.shard_states[offset_index].transition_digest: + shard_state = get_post_shard_state(shard_state, block) + if shard_state != transition.shard_states[offset_index]: return True return False @@ -152,145 +141,3 @@ def generate_custody_bit(subkey: BLSPubkey, block: ShardBlock) -> bool: # TODO ... ``` - -## Honest committee member behavior - -### Helper functions - -```python -def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedShardBlock]) -> SignedShardBlock: - # TODO: Let `winning_proposal` be the proposal with the largest number of total attestations from slots in - # `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing - # the first proposal locally seen. Do `proposals.append(winning_proposal)`. - return proposals[-1] # stub -``` - -```python -def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]: - return [hash_tree_root(proposal.message.body) for proposal in proposals] -``` - -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) - return choices -``` - -```python -def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, - slot: Shard, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Tuple[SignedShardBlock, ShardState]: - """ - Return ``proposal``, ``shard_state`` of the given ``slot``. - Note that this function doesn't change the state. - """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_state=shard_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: - block = ShardBlock(slot=slot) - proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] - else: - proposal = get_winning_proposal(beacon_state, choices) - - # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) - - return proposal, shard_state -``` - -```python -def get_shard_state_transition_result( - beacon_state: BeaconState, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True, -) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: - proposals = [] - shard_states = [] - shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): - proposal, shard_state = get_proposal_at_slot( - beacon_state=beacon_state, - shard_state=shard_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - shard_states.append(shard_state) - proposals.append(proposal) - - shard_data_roots = compute_shard_body_roots(proposals) - - return proposals, shard_states, shard_data_roots -``` - -### Make attestations - -Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks` since the latest successful crosslink for `shard` into the beacon chain. Let `beacon_state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`. - -```python -def get_shard_transition(beacon_state: BeaconState, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - start_slot = offset_slots[0] - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) - - assert len(proposals) > 0 - assert len(shard_data_roots) > 0 - - shard_block_lengths = [] - proposer_signatures = [] - for proposal in proposals: - shard_block_lengths.append(len(proposal.message.body)) - if proposal.signature != NO_SIGNATURE: - proposer_signatures.append(proposal.signature) - - if len(proposer_signatures) > 0: - proposer_signature_aggregate = bls.Aggregate(proposer_signatures) - else: - proposer_signature_aggregate = NO_SIGNATURE - - return ShardTransition( - start_slot=start_slot, - shard_block_lengths=shard_block_lengths, - shard_data_roots=shard_data_roots, - shard_states=shard_states, - proposer_signature_aggregate=proposer_signature_aggregate, - ) -``` diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md new file mode 100644 index 0000000000..ea69f03899 --- /dev/null +++ b/specs/phase1/validator.md @@ -0,0 +1,556 @@ +# Ethereum 2.0 Phase 0 -- Honest Validator + +**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 1](./), which describes the expected actions of a "validator" participating in the Ethereum 2.0 Phase 1 protocol. + +## Table of contents + + + + + + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Constants](#constants) + - [Misc](#misc) +- [Becoming a validator](#becoming-a-validator) +- [Beacon chain validator assignments](#beacon-chain-validator-assignments) + - [Lookahead](#lookahead) +- [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Preparing for a `BeaconBlock`](#preparing-for-a-beaconblock) + - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) + - [Custody slashings](#custody-slashings) + - [Custody key reveals](#custody-key-reveals) + - [Early derived secret reveals](#early-derived-secret-reveals) + - [Shard transitions](#shard-transitions) + - [Light client fields](#light-client-fields) + - [Packaging into a `SignedBeaconBlock`](#packaging-into-a-signedbeaconblock) + - [Attesting](#attesting) + - [`FullAttestationData`](#fullattestationdata) + - [`FullAttestation`](#fullattestation) + - [Timing](#timing) + - [Attestation data](#attestation-data) + - [Head shard root](#head-shard-root) + - [Shard transition](#shard-transition) + - [Construct attestation](#construct-attestation) + - [Attestation Aggregation](#attestation-aggregation) + - [Broadcast aggregate](#broadcast-aggregate) + - [`AggregateAndProof`](#aggregateandproof) + - [`SignedAggregateAndProof`](#signedaggregateandproof) + - [Light client committee](#light-client-committee) + - [Preparation](#preparation) + - [Light clent vote](#light-clent-vote) + - [Light client vote data](#light-client-vote-data) + - [`LightClientVoteData`](#lightclientvotedata) + - [Construct vote](#construct-vote) + - [`LightClientVote`](#lightclientvote) + - [Broadcast](#broadcast) + - [Light client vote aggregation](#light-client-vote-aggregation) + - [Aggregation selection](#aggregation-selection) + - [Construct aggregate](#construct-aggregate) + - [Broadcast aggregate](#broadcast-aggregate-1) + - [`LightAggregateAndProof`](#lightaggregateandproof) + - [`SignedLightAggregateAndProof`](#signedlightaggregateandproof) +- [How to avoid slashing](#how-to-avoid-slashing) + - [Custody slashing](#custody-slashing) + + + + +## Introduction + +This document represents the expected behavior of an "honest validator" with respect to Phase 1 of the Ethereum 2.0 protocol. This document does not distinguish between a "node" (i.e. the functionality of following and reading the beacon chain) and a "validator client" (i.e. the functionality of actively participating in consensus). The separation of concerns between these (potentially) two pieces of software is left as a design decision that is out of scope. + +A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof-of-work networks in which miners provide collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. + +## Prerequisites + +This document is an extension of the [Phase 0 -- Validator](../phase0/validator.md). All behaviors and definitions defined in the Phase 0 doc carry over unless explicitly noted or overridden. + +All terminology, constants, functions, and protocol mechanics defined in the [Phase 1 -- The Beacon Chain](./beacon-chain.md) and [Phase 1 -- Custody Game](./custody-game.md) docs are requisite for this document and used throughout. Please see the Phase 1 docs before continuing and use as a reference throughout. + +## Constants + +See constants from [Phase 0 validator guide](../phase0/validator.md#constants). + +### Misc + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT` | `2**3` (= 8) | validators | | +| `LIGHT_CLIENT_PREPARATION_EPOCHS` | `2**2` (= 4) | epochs | | + +## Becoming a validator + +Becoming a validator in Phase 1 is unchanged from Phase 0. See the [Phase 0 validator guide](../phase0/validator.md#becoming-a-validator) for details. + +## Beacon chain validator assignments + +Beacon chain validator assignments to beacon committees and beacon block proposal are unchanged from Phase 0. See the [Phase 0 validator guide](../phase0/validator.md#validator-assignments) for details. + +### Lookahead + +Lookahead for beacon committee assignments operates in the same manner as Phase 0, but committee members must join a shard block pubsub topic in addition to the committee attestation topic. + +Specifically _after_ finding stable peers of attestation subnets (see Phase 0) a validator should: +* Let `shard = compute_shard_from_committee_index(committe_index)` +* Subscribe to the pubsub topic `shard_{shard}_block` (attestation subnet peers should have this topic available). + +## Beacon chain responsibilities + +A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. + +These responsibilities are largely unchanged from Phase 0, but utilize the updated `SignedBeaconBlock`, `BeaconBlock`, `BeaconBlockBody`, `Attestation`, and `AttestationData` definitions found in Phase 1. Below notes only the additional and modified behavior with respect to Phase 0. + +Phase 1 adds light client committees and associated responsibilities, discussed [below](#light-client-committee). + +### Block proposal + +#### Preparing for a `BeaconBlock` + +`slot`, `proposer_index`, `parent_root` fields are unchanged. + +#### Constructing the `BeaconBlockBody` + +`randao_reveal`, `eth1_data`, and `graffiti` are unchanged. + +`proposer_slashings`, `deposits`, and `voluntary_exits` are unchanged. + +`attester_slashings` and `attestations` operate exactly as in Phase 0, but with new definitations of `AttesterSlashing` and `Attestation`, along with modified validation conditions found in `process_attester_slashing` and `process_attestation`. + +##### Custody slashings + +Up to `MAX_CUSTODY_SLASHINGS`, [`CustodySlashing`](./custody-game.md#custodyslashing) objects can be included in the `block`. The custody slashings must satisfy the verification conditions found in [custody slashings processing](./custody-game.md#custody-slashings). The validator receives a small "whistleblower" reward for each custody slashing included (THIS IS NOT CURRENTLY THE CASE BUT PROBABLY SHOULD BE). + +##### Custody key reveals + +Up to `MAX_CUSTODY_KEY_REVEALS`, [`CustodyKeyReveal`](./custody-game.md#custodykeyreveal) objects can be included in the `block`. The custody key reveals must satisfy the verification conditions found in [custody key reveal processing](./custody-game.md#custody-key-reveals). The validator receives a small reward for each custody key reveal included. + +##### Early derived secret reveals + +Up to `MAX_EARLY_DERIVED_SECRET_REVEALS`, [`EarlyDerivedSecretReveal`](./custody-game.md#earlyderivedsecretreveal) objects can be included in the `block`. The early derived secret reveals must satisfy the verification conditions found in [early derived secret reveal processing](./custody-game.md#custody-key-reveals). The validator receives a small "whistleblower" reward for each early derived secrete reveal included. + +##### Shard transitions + +Exactly `MAX_SHARDS` [`ShardTransition`](./beacon-chain#shardtransition) objects are included in the block. Default each to an empty `ShardTransition()`. Then for each committee assigned to the slot with an associated `committee_index` and `shard`, set `shard_transitions[shard] = full_transitions[winning_root]` if the committee had enough weight to form a crosslink this slot. + +Specifically: +* Call `shards, winning_roots = get_shard_winning_roots(state, block.slot, block.body.attestations)` +* Let `full_transitions` be a dictionary mapping from the `shard_transition_root`s found in `attestations` to the corresponding full `ShardTransition` +* Then for each `shard` and `winning_root` in `zip(shards, winning_roots)` set `shard_transitions[shard] = full_transitions[winning_root]` + +*Note*: The `state` passed into `get_shard_winning_roots` must be transitioned the slot of `block.slot` to run accurately due to the internal use of `get_online_validator_indices` and `is_on_time_attestation`. + +```python +def get_shard_winning_roots(state: BeaconState, + attestations: Sequence[Attestation]) -> Tuple[Sequence[Shard], Sequence[Root]]: + shards = [] + winning_roots = [] + online_indices = get_online_validator_indices(state) + committee_count = get_committee_count_at_slot(state, state.slot) + for committee_index in map(CommitteeIndex, range(committee_count)): + shard = compute_shard_from_committee_index(state, committee_index, state.slot) + # All attestations in the block for this committee/shard and are "on time" + shard_attestations = [ + attestation for attestation in attestations + if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index + ] + committee = get_beacon_committee(state, state.slot, committee_index) + + # Loop over all shard transition roots, looking for a winning root + shard_transition_roots = set([a.data.shard_transition_root for a in shard_attestations]) + for shard_transition_root in sorted(shard_transition_roots): + transition_attestations = [ + a for a in shard_attestations + if a.data.shard_transition_root == shard_transition_root + ] + transition_participants: Set[ValidatorIndex] = set() + for attestation in transition_attestations: + participants = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + transition_participants = transition_participants.union(participants) + + enough_online_stake = ( + get_total_balance(state, online_indices.intersection(transition_participants)) * 3 >= + get_total_balance(state, online_indices.intersection(committee)) * 2 + ) + if enough_online_stake: + shards.append(shard) + winning_roots.append(shard_transition_root) + break + + return shards, winning_roots +``` + +##### Light client fields + +First retrieve `best_aggregate` from `get_best_light_client_aggregate` where `aggregates` is a list of valid aggregated `LightClientVote`s for the previous slot. + +Then: +* Set `light_client_bits = best_aggregate.aggregation_bits` +* Set `light_client_signature = best_aggregate.signature` + +```python +def get_best_light_client_aggregate(block: BeaconBlock, + aggregates: Sequence[LightClientVote]) -> LightClientVote: + viable_aggregates = [ + aggregate for aggregate in aggregates + if ( + aggregate.data.slot == compute_previous_slot(block.slot) + and aggregate.data.beacon_block_root == block.parent_root + ) + ] + + return max( + viable_aggregates, + # Ties broken by lexicographically by hash_tree_root + key=lambda a: (len([i for i in a.aggregation_bits if i == 1]), hash_tree_root(a)), + default=LightClientVote(), + ) +``` + +#### Packaging into a `SignedBeaconBlock` + +Packaging into a `SignedBeaconBlock` is unchanged from Phase 0. + +### Attesting + +A validator is expected to create, sign, and broadcast an attestation during each epoch. + +Assignments and the core of this duty are unchanged from Phase 0. There are a few additional fields related to the assigned shard chain. + +The `Attestation` and `AttestationData` defined in the [Phase 1 Beacon Chain spec]() utilizes `shard_transition_root: Root` rather than a full `ShardTransition`. For the purposes of the validator and p2p layer, a modified `FullAttestationData` and containing `FullAttestation` are used to send the accompanying `ShardTransition` in its entirety. Note that due to the properties of SSZ `hash_tree_root`, the root and signatures of `AttestationData` and `FullAttestationData` are equivalent. + +#### `FullAttestationData` + +```python +class FullAttestationData(Container): + slot: Slot + index: CommitteeIndex + # LMD GHOST vote + beacon_block_root: Root + # FFG vote + source: Checkpoint + target: Checkpoint + # Current-slot shard block root + shard_head_root: Root + # Full shard transition + shard_transition: ShardTransition +``` + +#### `FullAttestation` + +```python +class FullAttestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: FullAttestationData + signature: BLSSignature +``` + +#### Timing + +Note the timing of when to create/broadcast is altered from Phase 1. + +A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid `BeaconBlock` from the expected beacon block proposer and a valid `ShardBlock` for the expected shard block proposer for the assigned `slot` or (b) one-half of the `slot` has transpired (`SECONDS_PER_SLOT / 2` seconds after the start of `slot`) -- whichever comes _first_. + +#### Attestation data + +`attestation_data` is constructed in the same manner as Phase 0 but uses `FullAttestationData` with the addition of two fields -- `shard_head_root` and `shard_transition`. + +- Let `head_block` be the result of running the fork choice during the assigned slot. +- Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. +- Let `shard_head_block` be the result of running the fork choice on the assigned shard chain during the assigned slot. +- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `shard_head_block` (i.e. the value of the shard fork choice store of `get_pending_shard_blocks(store, shard_store)`). + +*Note*: We assume that the fork choice only follows branches with valid `offset_slots` with respect to the most recent beacon state shard transition for the queried shard. + +##### Head shard root + +Set `attestation_data.shard_head_root = hash_tree_root(shard_head_block)`. + +##### Shard transition + +Set `shard_transition` to the value returned by `get_shard_transition(head_state, shard, shard_blocks)`. + +```python +def get_shard_transition_fields( + beacon_state: BeaconState, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock], + validate_signature: bool=True, +) -> Tuple[Sequence[uint64], Sequence[Root], Sequence[ShardState]]: + shard_states = [] + shard_data_roots = [] + shard_block_lengths = [] + + shard_state = beacon_state.shard_states[shard] + shard_block_slots = [shard_block.message.slot for shard_block in shard_blocks] + offset_slots = compute_offset_slots( + get_latest_slot_for_shard(beacon_state, shard), + Slot(beacon_state.slot + 1), + ) + for slot in offset_slots: + if slot in shard_block_slots: + shard_block = shard_blocks[shard_block_slots.index(slot)] + shard_data_roots.append(hash_tree_root(shard_block.message.body)) + else: + shard_block = SignedShardBlock(message=ShardBlock(slot=slot, shard=shard)) + shard_data_roots.append(Root()) + shard_state = get_post_shard_state(shard_state, shard_block.message) + shard_states.append(shard_state) + shard_block_lengths.append(len(shard_block.message.body)) + + return shard_block_lengths, shard_data_roots, shard_states +``` + +```python +def get_shard_transition(beacon_state: BeaconState, + shard: Shard, + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + offset_slots = compute_offset_slots( + get_latest_slot_for_shard(beacon_state, shard), + Slot(beacon_state.slot + 1), + ) + shard_block_lengths, shard_data_roots, shard_states = ( + get_shard_transition_fields(beacon_state, shard, shard_blocks) + ) + + if len(shard_blocks) > 0: + proposer_signatures = [shard_block.signature for shard_block in shard_blocks] + proposer_signature_aggregate = bls.Aggregate(proposer_signatures) + else: + proposer_signature_aggregate = NO_SIGNATURE + + return ShardTransition( + start_slot=offset_slots[0], + shard_block_lengths=shard_block_lengths, + shard_data_roots=shard_data_roots, + shard_states=shard_states, + proposer_signature_aggregate=proposer_signature_aggregate, + ) +``` + +#### Construct attestation + +Next, the validator creates `attestation`, a `FullAttestation` as defined above. + +`attestation.data`, `attestation.aggregation_bits`, and `attestation.signature` are unchanged from Phase 0. But safety/validity in signing the message is premised upon calculation of the "custody bit" [TODO]. + +### Attestation Aggregation + +Some validators are selected to locally aggregate attestations with a similar `attestation_data` to their constructed `attestation` for the assigned `slot`. + +Aggregation selection and the core of this duty are largely unchanged from Phase 0. Any additional components or changes are noted. + +#### Broadcast aggregate + +Note the timing of when to broadcast aggregates is altered in Phase 1+. + +If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) three-fourths of the way through the `slot`-that is, `SECONDS_PER_SLOT * 3 / 4` seconds after the start of `slot`. + +##### `AggregateAndProof` + +`AggregateAndProof` is unchanged other than the contained `Attestation`. + +```python +class AggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` + +##### `SignedAggregateAndProof` + +`AggregateAndProof` is unchanged other than the contained `AggregateAndProof`. + +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` + +### Light client committee + +In addition to the core beacon chain responsibilities, Phase 1 adds an additional role -- the Light Client Committee -- to aid in light client functionality. + +Validators serve on the light client committee for `LIGHT_CLIENT_COMMITTEE_PERIOD` epochs and the assignment to be on a committee is known `LIGHT_CLIENT_COMMITTEE_PERIOD` epochs in advance. + +#### Preparation + +When `get_current_epoch(state) % LIGHT_CLIENT_COMMITTEE_PERIOD == LIGHT_CLIENT_COMMITTEE_PERIOD - LIGHT_CLIENT_PREPARATION_EPOCHS` each validator must check if they are in the next period light client committee by calling `is_in_next_light_client_committee()`. + +If the validator is in the next light client committee, they must join the `light_client_votes` pubsub topic to begin duties at the start of the next period. + +```python +def is_in_next_light_client_committee(state: BeaconState, index: ValidatorIndex) -> bool: + next_committee = get_light_client_committee(state, get_current_epoch(state) + LIGHT_CLIENT_COMMITTEE_PERIOD) + return index in next_committee +``` + +#### Light clent vote + +During a period of epochs that the validator is a part of the light client committee (`validator_index in get_light_client_committee(state, epoch)`), the validator creates and broadcasts a `LightClientVote` at each slot. + +A validator should create and broadcast the `light_client_vote` to the `light_client_votes` pubsub topic when either (a) the validator has received a valid block from the expected block proposer for the current `slot` or (b) two-thirds of the `slot` have transpired (`SECONDS_PER_SLOT / 3` seconds after the start of `slot`) -- whichever comes _first_. + +- Let `light_client_committee = get_light_client_committee(state, compute_epoch_at_slot(slot))` + +##### Light client vote data + +First the validator constructs `light_client_vote_data`, a [`LightClientVoteData`](#lightclientvotedata) object. + +* Let `head_block` be the result of running the fork choice during the assigned slot. +* Set `light_client_vote.slot = slot`. +* Set `light_client_vote.beacon_block_root = hash_tree_root(head_block)`. + +###### `LightClientVoteData` + +```python +class LightClientVoteData(Container): + slot: Slot + beacon_block_root: Root +``` + +##### Construct vote + +Then the validator constructs `light_client_vote`, a [`LightClientVote`](#lightclientvote) object. + +* Set `light_client_vote.data = light_client_vote_data`. +* Set `light_client_vote.aggregation_bits` to be a `Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE]`, where the bit of the index of the validator in the `light_client_committee` is set to `0b1` and all other bits are are set to `0b0`. +* Set `light_client_vote.signature = vote_signature` where `vote_signature` is obtained from: + +```python +def get_light_client_vote_signature(state: BeaconState, + light_client_vote_data: LightClientVoteData, + privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_LIGHT_CLIENT, compute_epoch_at_slot(light_client_vote_data.slot)) + signing_root = compute_signing_root(light_client_vote_data, domain) + return bls.Sign(privkey, signing_root) +``` + +###### `LightClientVote` + +```python +class LightClientVote(Container): + data: LightClientVoteData + aggregation_bits: Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE] + signature: BLSSignature +``` + +##### Broadcast + +Finally, the validator broadcasts `light_client_vote` to the `light_client_votes` pubsub topic. + +#### Light client vote aggregation + +Some validators in the light client committee are selected to locally aggregate light client votes with a similar `light_client_vote_data` to their constructed `light_client_vote` for the assigned `slot`. + +#### Aggregation selection + +A validator is selected to aggregate based upon the return value of `is_light_client_aggregator()`. + +```python +def get_light_client_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_LIGHT_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` + +```python +def is_light_client_aggregator(state: BeaconState, slot: Slot, slot_signature: BLSSignature) -> bool: + committee = get_light_client_committee(state, compute_epoch_at_slot(slot)) + modulo = max(1, len(committee) // TARGET_LIGHT_CLIENT_AGGREGATORS_PER_SLOT) + return bytes_to_int(hash(slot_signature)[0:8]) % modulo == 0 +``` + +#### Construct aggregate + +If the validator is selected to aggregate (`is_light_client_aggregator()`), they construct an aggregate light client vote via the following. + +Collect `light_client_votes` seen via gossip during the `slot` that have an equivalent `light_client_vote_data` to that constructed by the validator, and create a `aggregate_light_client_vote: LightClientVote` with the following fields. + +* Set `aggregate_light_client_vote.data = light_client_vote_data` where `light_client_vote_data` is the `LightClientVoteData` object that is the same for each individual light client vote being aggregated. +* Set `aggregate_light_client_vote.aggregation_bits` to be a `Bitvector[LIGHT_CLIENT_COMMITTEE_SIZE]`, where each bit set from each individual light client vote is set to `0b1`. +* Set `aggregate_light_client_vote.signature = aggregate_light_client_signature` where `aggregate_light_client_signature` is obtained from `get_aggregate_light_client_signature`. + +```python +def get_aggregate_light_client_signature(light_client_votes: Sequence[LightClientVote]) -> BLSSignature: + signatures = [light_client_vote.signature for light_client_vote in light_client_votes] + return bls.Aggregate(signatures) +``` + +#### Broadcast aggregate + +If the validator is selected to aggregate (`is_light_client_aggregator`), then they broadcast their best aggregate light client vote as a `SignedLightAggregateAndProof` to the global aggregate light client vote channel (`aggregate_light_client_votes`) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. + +Selection proofs are provided in `LightAggregateAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. + +`LightAggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedLightAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. + +First, `light_aggregate_and_proof = get_light_aggregate_and_proof(state, validator_index, aggregate_light_client_vote, privkey)` is constructed. + +```python +def get_light_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: LightClientVote, + privkey: int) -> LightAggregateAndProof: + return LightAggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_light_client_slot_signature(state, aggregate.data.slot, privkey), + ) +``` + +Then `signed_light_aggregate_and_proof = SignedLightAggregateAndProof(message=light_aggregate_and_proof, signature=signature)` is constructed and broadast. Where `signature` is obtained from: + +```python +def get_light_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: LightAggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_LIGHT_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` + +##### `LightAggregateAndProof` + +```python +class LightAggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: LightClientVote + selection_proof: BLSSignature +``` + +##### `SignedLightAggregateAndProof` + +```python +class SignedLightAggregateAndProof(Container): + message: LightAggregateAndProof + signature: BLSSignature +``` + +## How to avoid slashing + +Proposer and Attester slashings described in Phase 0 remain in place with the + addition of the following. + +### Custody slashing + +To avoid custody slashings, the attester must never sign any shard transition for which the custody bit is one. The custody bit is computed using the custody secret: + +```python +def get_custody_secret(state: BeaconState, + validator_index: ValidatorIndex, + privkey: int, + epoch: Epoch=None) -> BLSSignature: + if epoch is None: + epoch = get_current_epoch(state) + period = get_custody_period_for_validator(validator_index, epoch) + epoch_to_sign = get_randao_epoch_for_custody_period(period, validator_index) + domain = get_domain(state, DOMAIN_RANDAO, epoch_to_sign) + signing_root = compute_signing_root(Epoch(epoch_to_sign), domain) + return bls.Sign(privkey, signing_root) +``` + +Note that the valid custody secret is always the one for the **attestation target epoch**, not to be confused with the epoch in which the shard block was generated. While they are the same most of the time, getting this wrong at custody epoch boundaries would result in a custody slashing. diff --git a/tests/core/pyspec/README.md b/tests/core/pyspec/README.md index a9ee80105b..f7092dbce9 100644 --- a/tests/core/pyspec/README.md +++ b/tests/core/pyspec/README.md @@ -55,6 +55,11 @@ Run the test command from the `tests/core/pyspec` directory: pytest --config=minimal eth2spec ``` +Options: +- `--config`, to change the config. Defaults to `minimal`, can be set to `mainnet`, or other configs from the configs directory. +- `--disable-bls`, to disable BLS (only for tests that can run without) +- `--bls-type`, `milagro` or `py_ecc` (default) + ### How to view code coverage report Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report. diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index d33c3a2128..aac2dacab4 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.12.0 \ No newline at end of file +0.12.1 \ No newline at end of file diff --git a/tests/core/pyspec/eth2spec/config/config_util.py b/tests/core/pyspec/eth2spec/config/config_util.py index c43c1521bc..1977e5b054 100644 --- a/tests/core/pyspec/eth2spec/config/config_util.py +++ b/tests/core/pyspec/eth2spec/config/config_util.py @@ -1,8 +1,9 @@ -from ruamel.yaml import YAML +import os from pathlib import Path -from os.path import join from typing import Dict, Any +from ruamel.yaml import YAML + config: Dict[str, Any] = {} @@ -35,11 +36,19 @@ def load_config_file(configs_dir: str, presets_name: str) -> Dict[str, Any]: :param presets_name: The name of the presets. (lowercase snake_case) :return: Dictionary, mapping of constant-name -> constant-value """ - path = Path(join(configs_dir, presets_name + '.yaml')) - yaml = YAML(typ='base') - loaded = yaml.load(path) + present_dir = Path(configs_dir) / presets_name + _, _, config_files = next(os.walk(present_dir)) + config_files.sort() + loaded_config = {} + for config_file_name in config_files: + yaml = YAML(typ='base') + path = present_dir / config_file_name + loaded = yaml.load(path) + loaded_config.update(loaded) + assert loaded_config != {} + out: Dict[str, Any] = dict() - for k, v in loaded.items(): + for k, v in loaded_config.items(): if isinstance(v, list): # Clean up integer values. YAML parser renders lists of ints as list of str out[k] = [int(item) if item.isdigit() else item for item in v] diff --git a/tests/core/pyspec/eth2spec/test/conftest.py b/tests/core/pyspec/eth2spec/test/conftest.py index 01187b05fc..01c974ae0d 100644 --- a/tests/core/pyspec/eth2spec/test/conftest.py +++ b/tests/core/pyspec/eth2spec/test/conftest.py @@ -1,6 +1,6 @@ from eth2spec.config import config_util -from eth2spec.test.context import reload_specs - +from eth2spec.test import context +from eth2spec.utils import bls as bls_utils # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. @@ -27,7 +27,16 @@ def ignore(): def pytest_addoption(parser): parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + "--config", action="store", type=str, default="minimal", + help="config: make the pyspec use the specified configuration" + ) + parser.addoption( + "--disable-bls", action="store_true", + help="bls-default: make tests that are not dependent on BLS run without BLS" + ) + parser.addoption( + "--bls-type", action="store", type=str, default="py_ecc", choices=["py_ecc", "milagro"], + help="bls-type: use 'pyecc' or 'milagro' implementation for BLS" ) @@ -36,4 +45,22 @@ def config(request): config_name = request.config.getoption("--config") config_util.prepare_config('../../../configs/', config_name) # now that the presets are loaded, reload the specs to apply them - reload_specs() + context.reload_specs() + + +@fixture(autouse=True) +def bls_default(request): + disable_bls = request.config.getoption("--disable-bls") + if disable_bls: + context.DEFAULT_BLS_ACTIVE = False + + +@fixture(autouse=True) +def bls_type(request): + bls_type = request.config.getoption("--bls-type") + if bls_type == "py_ecc": + bls_utils.bls = bls_utils.py_ecc_bls + elif bls_type == "milagro": + bls_utils.bls = bls_utils.milagro_bls + else: + raise Exception(f"unrecognized bls type: {bls_type}") diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 20214908e2..0d953cf241 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -167,14 +167,15 @@ def entry(*args, **kw): return entry -# BLS is turned off by default *for performance purposes during TESTING*. +# BLS is turned on by default, it can be disabled in tests by overriding this, or using `--disable-bls`. +# *This is for performance purposes during TESTING, DO NOT DISABLE IN PRODUCTION*. # The runner of the test can indicate the preferred setting (test generators prefer BLS to be ON). # - Some tests are marked as BLS-requiring, and ignore this setting. # (tests that express differences caused by BLS, e.g. invalid signatures being rejected) # - Some other tests are marked as BLS-ignoring, and ignore this setting. # (tests that are heavily performance impacted / require unsigned state transitions) # - Most tests respect the BLS setting. -DEFAULT_BLS_ACTIVE = False +DEFAULT_BLS_ACTIVE = True def spec_test(fn): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f7..f9a7398047 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): @@ -183,7 +155,7 @@ def test_filtered_block_tree(spec, state): for i in range(spec.SLOTS_PER_EPOCH): slot = rogue_block.slot + i for index in range(spec.get_committee_count_at_slot(non_viable_state, slot)): - attestation = get_valid_attestation(spec, non_viable_state, rogue_block.slot + i, index) + attestation = get_valid_attestation(spec, non_viable_state, slot, index, signed=True) attestations.append(attestation) # tick time forward to be able to include up to the latest attestation diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index b2d33d0aa7..2c50544d84 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -16,20 +16,22 @@ def run_on_attestation(spec, state, store, attestation, valid=True): indexed_attestation = spec.get_indexed_attestation(state, attestation) spec.on_attestation(store, attestation) + sample_index = indexed_attestation.attesting_indices[0] if spec.fork == PHASE0: - sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: - attesting_indices = [ - index for i, index in enumerate(indexed_attestation.committee) - if attestation.aggregation_bits[i] - ] - sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.shard_head_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 0000000000..24eeaedbee --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,132 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation +from eth2spec.test.helpers.shard_block import ( + build_shard_block, + get_shard_transitions, + get_committee_index_of_shard, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard_store, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message + + +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): + shard = shard_store.shard + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_parent_state.slot + shard_block = build_shard_block( + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pending_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pending_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block + shard_transitions = get_shard_transitions( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + ) + shard_transition = shard_transitions[shard] + attestation = get_valid_on_time_attestation( + spec, + state, + index=committee_index, + shard_transition=shard_transition, + signed=False, + ) + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + + # Clear buffer + shard_blocks_buffer.clear() + + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! + add_block_to_store(spec, store, signed_beacon_block) + assert spec.get_head(store) == beacon_block.hash_tree_root() + + # On shard block at transitioned `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) + + return has_shard_committee + + +@with_all_phases_except([PHASE0]) +@spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here + state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) + + # Initialization + store = spec.get_forkchoice_store(state) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + shard_store = spec.get_forkchoice_shard_store(state, shard) + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 + shard_blocks_buffer = [] + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_committee_counter -= 1 diff --git a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py index 8828213377..faade2d17e 100644 --- a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py @@ -21,7 +21,7 @@ def test_initialize_beacon_state_from_eth1(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY + assert state.genesis_time == eth1_timestamp + spec.GENESIS_DELAY assert len(state.validators) == deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == deposit_count @@ -57,7 +57,7 @@ def test_initialize_beacon_state_some_small_balances(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY + assert state.genesis_time == eth1_timestamp + spec.GENESIS_DELAY assert len(state.validators) == small_deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == len(deposits) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 79f752411a..23eba34da8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,12 +1,14 @@ +from lru import LRU + from typing import List -from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 -from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot, transition_to +from eth2spec.test.context import expect_assertion_error, PHASE1 +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist -from lru import LRU def run_attestation_processing(spec, state, attestation, valid=True): @@ -78,41 +80,22 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if spec.fork == PHASE1: if shard_transition is not None: lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 - attestation_data.head_shard_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] + attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - # No shard transition + # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, []) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 - attestation_data.head_shard_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] + attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - attestation_data.head_shard_root = state.shard_states[shard].transition_digest + attestation_data.shard_head_root = state.shard_states[shard].latest_block_root attestation_data.shard_transition_root = spec.Root() return attestation_data -def convert_to_valid_on_time_attestation(spec, state, attestation, signed=False): - shard = spec.get_shard(state, attestation) - offset_slots = spec.compute_offset_slots( - spec.get_latest_slot_for_shard(state, shard), - attestation.data.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY, - ) - for _ in offset_slots: - attestation.custody_bits_blocks.append( - Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE]([0 for _ in attestation.aggregation_bits]) - ) - - if signed: - sign_attestation(spec, state, attestation) - - return attestation - - def get_valid_on_time_attestation(spec, state, slot=None, index=None, shard_transition=None, signed=False): ''' Construct on-time attestation for next slot @@ -133,7 +116,7 @@ def get_valid_on_time_attestation(spec, state, slot=None, index=None, shard_tran ) -def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False): +def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False, shard_transition=None): ''' Construct on-time attestation for next slot ''' @@ -142,7 +125,8 @@ def get_valid_late_attestation(spec, state, slot=None, index=None, signed=False) if index is None: index = 0 - return get_valid_attestation(spec, state, slot=slot, index=index, signed=signed, on_time=False) + return get_valid_attestation(spec, state, slot=slot, index=index, + signed=signed, on_time=False, shard_transition=shard_transition) def get_valid_attestation(spec, @@ -179,9 +163,6 @@ def get_valid_attestation(spec, # fill the attestation with (optionally filtered) participants, and optionally sign it fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) - if spec.fork == PHASE1 and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) - return attestation @@ -201,40 +182,9 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List def sign_indexed_attestation(spec, state, indexed_attestation): - if spec.fork == PHASE0: - participants = indexed_attestation.attesting_indices - data = indexed_attestation.data - indexed_attestation.signature = sign_aggregate_attestation(spec, state, data, participants) - else: - participants = spec.get_indices_from_committee( - indexed_attestation.committee, - indexed_attestation.attestation.aggregation_bits, - ) - data = indexed_attestation.attestation.data - indexed_attestation.attestation.signature = sign_aggregate_attestation(spec, state, data, participants) - - -def sign_on_time_attestation(spec, state, attestation): - if not any(attestation.custody_bits_blocks): - sign_attestation(spec, state, attestation) - return - - committee = spec.get_beacon_committee(state, attestation.data.slot, attestation.data.index) - signatures = [] - for block_index, custody_bits in enumerate(attestation.custody_bits_blocks): - for participant, abit, cbit in zip(committee, attestation.aggregation_bits, custody_bits): - if not abit: - continue - signatures.append(get_attestation_custody_signature( - spec, - state, - attestation.data, - block_index, - cbit, - privkeys[participant] - )) - - attestation.signature = bls.Aggregate(signatures) + participants = indexed_attestation.attesting_indices + data = indexed_attestation.data + indexed_attestation.signature = sign_aggregate_attestation(spec, state, data, participants) def get_attestation_custody_signature(spec, state, attestation_data, block_index, bit, privkey): @@ -251,10 +201,6 @@ def get_attestation_custody_signature(spec, state, attestation_data, block_index def sign_attestation(spec, state, attestation): - if spec.fork == PHASE1 and any(attestation.custody_bits_blocks): - sign_on_time_attestation(spec, state, attestation) - return - participants = spec.get_attesting_indices( state, attestation.data, @@ -314,7 +260,17 @@ def next_epoch_with_attestations(spec, committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): for index in range(committees_per_slot): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) + block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + cur_attestation = get_valid_attestation( + spec, post_state, slot_to_attest, + shard_transition=shard_transition, index=index, signed=True, on_time=True + ) block.body.attestations.append(cur_attestation) if fill_prev_epoch: @@ -325,9 +281,6 @@ def next_epoch_with_attestations(spec, spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) block.body.attestations.append(prev_attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, post_state, block) - signed_block = state_transition_and_sign_block(spec, post_state, block) signed_blocks.append(signed_block) @@ -393,14 +346,3 @@ def cached_prepare_state_with_attestations(spec, state): # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) - - -def fill_block_shard_transitions_by_attestations(spec, state, block): - block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS - for attestation in block.body.attestations: - shard = spec.get_shard(state, attestation) - if attestation.data.slot == state.slot: - temp_state = state.copy() - transition_to(spec, temp_state, slot=block.slot) - shard_transition = spec.get_shard_transition(temp_state, shard, []) - block.body.shard_transitions[shard] = shard_transition diff --git a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py index e743ca8ff6..43763895fc 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attester_slashings.py @@ -1,4 +1,3 @@ -from eth2spec.test.context import PHASE1 from eth2spec.test.helpers.attestations import get_valid_attestation, sign_attestation, sign_indexed_attestation @@ -41,34 +40,19 @@ def get_indexed_attestation_participants(spec, indexed_att): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. """ - if spec.fork == PHASE1: - return list(spec.get_indices_from_committee( - indexed_att.committee, - indexed_att.attestation.aggregation_bits, - )) - else: - return list(indexed_att.attesting_indices) + return list(indexed_att.attesting_indices) def set_indexed_attestation_participants(spec, indexed_att, participants): """ Wrapper around index-attestation to return the list of participant indices, regardless of spec phase. """ - if spec.fork == PHASE1: - indexed_att.attestation.aggregation_bits = [bool(i in participants) for i in indexed_att.committee] - else: - indexed_att.attesting_indices = participants + indexed_att.attesting_indices = participants def get_attestation_1_data(spec, att_slashing): - if spec.fork == PHASE1: - return att_slashing.attestation_1.attestation.data - else: - return att_slashing.attestation_1.data + return att_slashing.attestation_1.data def get_attestation_2_data(spec, att_slashing): - if spec.fork == PHASE1: - return att_slashing.attestation_2.attestation.data - else: - return att_slashing.attestation_2.data + return att_slashing.attestation_2.data diff --git a/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py b/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py deleted file mode 100644 index ea5da89d96..0000000000 --- a/tests/core/pyspec/eth2spec/test/helpers/crosslinks.py +++ /dev/null @@ -1,28 +0,0 @@ -from eth2spec.test.context import expect_assertion_error - - -def run_crosslinks_processing(spec, state, shard_transitions, attestations, valid=True): - """ - Run ``process_attestation``, yielding: - - pre-state ('pre') - - shard_transitions ('shard_transitions') - - attestations ('attestations') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # yield pre-state - yield 'pre', state - yield 'shard_transitions', shard_transitions - yield 'attestations', attestations - - # If the attestation is invalid, processing is aborted, and there is no post-state. - if not valid: - expect_assertion_error(lambda: spec.process_crosslinks(state, shard_transitions, attestations)) - yield 'post', None - return - - # process crosslinks - spec.process_crosslinks(state, shard_transitions, attestations) - - # yield post-state - yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 7c51675cdc..f63a07099d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -1,10 +1,7 @@ from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls -from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, Bitvector -from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.merkle_minimal import get_merkle_tree, get_merkle_proof -from remerkleable.core import pack_bits_to_chunks -from remerkleable.tree import subtree_fill_to_contents, get_depth +from eth2spec.utils.ssz.ssz_typing import Bitlist, ByteVector, ByteList +from remerkleable.tree import gindex_bit_iter BYTES_PER_CHUNK = 32 @@ -37,9 +34,10 @@ def get_valid_early_derived_secret_reveal(spec, state, epoch=None): ) -def get_valid_custody_key_reveal(spec, state, period=None): +def get_valid_custody_key_reveal(spec, state, period=None, validator_index=None): current_epoch = spec.get_current_epoch(state) - revealer_index = spec.get_active_validator_indices(state, current_epoch)[0] + revealer_index = (spec.get_active_validator_indices(state, current_epoch)[0] + if validator_index is None else validator_index) revealer = state.validators[revealer_index] if period is None: @@ -61,38 +59,54 @@ def bitlist_from_int(max_len, num_bits, n): return Bitlist[max_len](*[(n >> i) & 0b1 for i in range(num_bits)]) -def get_valid_bit_challenge(spec, state, attestation, invalid_custody_bit=False): +def get_valid_custody_slashing(spec, state, attestation, shard_transition, custody_secret, data, data_index=0): beacon_committee = spec.get_beacon_committee( state, attestation.data.slot, - attestation.data.crosslink.shard, + attestation.data.index, ) - responder_index = beacon_committee[0] - challenger_index = beacon_committee[-1] + malefactor_index = beacon_committee[0] + whistleblower_index = beacon_committee[-1] + + slashing = spec.CustodySlashing( + data_index=data_index, + malefactor_index=malefactor_index, + malefactor_secret=custody_secret, + whistleblower_index=whistleblower_index, + shard_transition=shard_transition, + attestation=attestation, + data=data, + ) + slashing_domain = spec.get_domain(state, spec.DOMAIN_CUSTODY_BIT_SLASHING) + slashing_root = spec.compute_signing_root(slashing, slashing_domain) - epoch = spec.get_randao_epoch_for_custody_period(attestation.data.target.epoch, - responder_index) + signed_slashing = spec.SignedCustodySlashing( + message=slashing, + signature=bls.Sign(privkeys[whistleblower_index], slashing_root) + ) - # Generate the responder key - domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch) - signing_root = spec.compute_signing_root(spec.Epoch(epoch), domain) - responder_key = bls.Sign(privkeys[responder_index], signing_root) + return signed_slashing - chunk_count = spec.get_custody_chunk_count(attestation.data.crosslink) - chunk_bits = bitlist_from_int(spec.MAX_CUSTODY_CHUNKS, chunk_count, 0) +def get_valid_chunk_challenge(spec, state, attestation, shard_transition, data_index=None, chunk_index=None): + crosslink_committee = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index + ) + responder_index = crosslink_committee[0] + data_index = len(shard_transition.shard_block_lengths) - 1 if not data_index else data_index - n = 0 - while spec.get_chunk_bits_root(chunk_bits) == attestation.custody_bits[0] ^ invalid_custody_bit: - chunk_bits = bitlist_from_int(spec.MAX_CUSTODY_CHUNKS, chunk_count, n) - n += 1 + chunk_count = (shard_transition.shard_block_lengths[data_index] + + spec.BYTES_PER_CUSTODY_CHUNK - 1) // spec.BYTES_PER_CUSTODY_CHUNK + chunk_index = chunk_count - 1 if not chunk_index else chunk_index - return spec.CustodyBitChallenge( + return spec.CustodyChunkChallenge( responder_index=responder_index, attestation=attestation, - challenger_index=challenger_index, - responder_key=responder_key, - chunk_bits=chunk_bits, + chunk_index=chunk_index, + data_index=data_index, + shard_transition=shard_transition, ) @@ -102,50 +116,90 @@ def custody_chunkify(spec, x): return chunks -def get_valid_custody_response(spec, state, bit_challenge, custody_data, challenge_index, invalid_chunk_bit=False): +def build_proof(anchor, leaf_index): + if leaf_index <= 1: + return [] # Nothing to prove / invalid index + node = anchor + proof = [] + # Walk down, top to bottom to the leaf + bit_iter, _ = gindex_bit_iter(leaf_index) + for bit in bit_iter: + # Always take the opposite hand for the proof. + # 1 = right as leaf, thus get left + if bit: + proof.append(node.get_left().merkle_root()) + node = node.get_right() + else: + proof.append(node.get_right().merkle_root()) + node = node.get_left() + + return list(reversed(proof)) + + +def get_block_data_merkle_root(data_as_bytelist): + # To get the Merkle root of the block data, we need the Merkle root without the length Mixing + # The below implements this in the Remerkleable framework + return data_as_bytelist.get_backing().get_left().merkle_root() + + +def get_valid_custody_chunk_response(spec, state, chunk_challenge, block_length, challenge_index, + invalid_chunk_data=False): + custody_data = get_custody_test_vector(block_length) + custody_data_block = ByteList[spec.MAX_SHARD_BLOCK_SIZE](custody_data) chunks = custody_chunkify(spec, custody_data) - chunk_index = len(chunks) - 1 - chunk_bit = spec.get_custody_chunk_bit(bit_challenge.responder_key, chunks[chunk_index]) + chunk_index = chunk_challenge.chunk_index - while chunk_bit == bit_challenge.chunk_bits[chunk_index] ^ invalid_chunk_bit: - chunk_index -= 1 - chunk_bit = spec.get_custody_chunk_bit(bit_challenge.responder_key, chunks[chunk_index]) + data_branch = build_proof(custody_data_block.get_backing().get_left(), chunk_index + 2**spec.CUSTODY_RESPONSE_DEPTH) - chunks_hash_tree_roots = [hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunk)) for chunk in chunks] - chunks_hash_tree_roots += [ - hash_tree_root(ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](b"\0" * spec.BYTES_PER_CUSTODY_CHUNK)) - for i in range(2 ** spec.ceillog2(len(chunks)) - len(chunks))] - data_tree = get_merkle_tree(chunks_hash_tree_roots) - - data_branch = get_merkle_proof(data_tree, chunk_index) + return spec.CustodyChunkResponse( + challenge_index=challenge_index, + chunk_index=chunk_index, + chunk=ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunks[chunk_index]), + branch=data_branch, + ) - bitlist_chunk_index = chunk_index // BYTES_PER_CHUNK - print(bitlist_chunk_index) - bitlist_chunk_nodes = pack_bits_to_chunks(bit_challenge.chunk_bits) - bitlist_tree = subtree_fill_to_contents(bitlist_chunk_nodes, get_depth(spec.MAX_CUSTODY_CHUNKS)) - print(bitlist_tree) - bitlist_chunk_branch = None # TODO; extract proof from merkle tree - bitlist_chunk_index = chunk_index // 256 +def get_custody_test_vector(bytelength, offset=0): + ints = bytelength // 4 + 1 + return (b"".join((i + offset).to_bytes(4, "little") for i in range(ints)))[:bytelength] - chunk_bits_leaf = Bitvector[256](bit_challenge.chunk_bits[bitlist_chunk_index * 256: - (bitlist_chunk_index + 1) * 256]) - return spec.CustodyResponse( - challenge_index=challenge_index, - chunk_index=chunk_index, - chunk=ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunks[chunk_index]), - data_branch=data_branch, - chunk_bits_branch=bitlist_chunk_branch, - chunk_bits_leaf=chunk_bits_leaf, +def get_sample_shard_transition(spec, start_slot, block_lengths): + b = [get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) + for x in block_lengths] + shard_transition = spec.ShardTransition( + start_slot=start_slot, + shard_block_lengths=block_lengths, + shard_data_roots=b, + shard_states=[spec.Root() for x in block_lengths], + proposer_signature_aggregate=spec.BLSSignature(), ) + return shard_transition + + +def get_custody_secret(spec, state, validator_index, epoch=None): + period = spec.get_custody_period_for_validator(validator_index, epoch if epoch is not None + else spec.get_current_epoch(state)) + epoch_to_sign = spec.get_randao_epoch_for_custody_period(period, validator_index) + domain = spec.get_domain(state, spec.DOMAIN_RANDAO, epoch_to_sign) + signing_root = spec.compute_signing_root(spec.Epoch(epoch_to_sign), domain) + return bls.Sign(privkeys[validator_index], signing_root) -def get_custody_test_vector(bytelength): - ints = bytelength // 4 - return b"".join(i.to_bytes(4, "little") for i in range(ints)) +def get_custody_slashable_test_vector(spec, custody_secret, length, slashable=True): + test_vector = get_custody_test_vector(length) + offset = 0 + while spec.compute_custody_bit(custody_secret, test_vector) != slashable: + offset += 1 + test_vector = test_vector = get_custody_test_vector(length, offset) + return test_vector -def get_custody_merkle_root(data): - return None # get_merkle_tree(chunkify(data))[-1][0] +def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, custody_secret, slashable=True): + shard_transition = get_sample_shard_transition(spec, start_slot, block_lengths) + slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, + block_lengths[0], slashable=slashable) + block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) + shard_transition.shard_data_roots[0] = get_block_data_merkle_root(block_data) + return shard_transition, slashable_test_vector diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 0000000000..04e36ea849 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) diff --git a/tests/core/pyspec/eth2spec/test/helpers/rewards.py b/tests/core/pyspec/eth2spec/test/helpers/rewards.py index d62fee6ce9..c11ba1ec1d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/rewards.py +++ b/tests/core/pyspec/eth2spec/test/helpers/rewards.py @@ -1,4 +1,5 @@ from random import Random +from lru import LRU from eth2spec.phase0 import spec as spec_phase0 from eth2spec.test.helpers.attestations import cached_prepare_state_with_attestations @@ -150,7 +151,6 @@ def run_get_inactivity_penalty_deltas(spec, state): matching_attestations = spec.get_matching_target_attestations(state, spec.get_previous_epoch(state)) matching_attesting_indices = spec.get_unslashed_attesting_indices(state, matching_attestations) - finality_delay = spec.get_previous_epoch(state) - state.finalized_checkpoint.epoch eligible_indices = spec.get_eligible_validator_indices(state) for index in range(len(state.validators)): assert rewards[index] == 0 @@ -158,8 +158,9 @@ def run_get_inactivity_penalty_deltas(spec, state): assert penalties[index] == 0 continue - if finality_delay > spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY: - base_penalty = spec.BASE_REWARDS_PER_EPOCH * spec.get_base_reward(state, index) + if spec.is_in_inactivity_leak(state): + base_reward = spec.get_base_reward(state, index) + base_penalty = spec.BASE_REWARDS_PER_EPOCH * base_reward - spec.get_proposer_reward(state, index) if not has_enough_for_reward(spec, state, index): assert penalties[index] == 0 elif index in matching_attesting_indices: @@ -170,6 +171,39 @@ def run_get_inactivity_penalty_deltas(spec, state): assert penalties[index] == 0 +def transition_state_to_leak(spec, state, epochs=None): + if epochs is None: + epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + + for _ in range(epochs): + next_epoch(spec, state) + + +_cache_dict = LRU(size=10) + + +def leaking(epochs=None): + + def deco(fn): + def entry(*args, spec, state, **kw): + # If the pre-state is not already known in the LRU, then take it, + # transition it to leak, and put it in the LRU. + # The input state is likely already cached, so the hash-tree-root does not affect speed. + key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) + global _cache_dict + if key not in _cache_dict: + transition_state_to_leak(spec, state, epochs=epochs) + _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. + + # Take an entry out of the LRU. + # No copy is necessary, as we wrap the immutable backing with a new view. + state = spec.BeaconState(backing=_cache_dict[key]) + return fn(*args, spec=spec, state=state, **kw) + return entry + return deco + + def set_some_new_deposits(spec, state, rng): num_validators = len(state.validators) # Set ~1/10 to just recently deposited diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index ef65d24279..ddf66f6c23 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -23,21 +21,24 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, + shard=shard, proposer_index=proposer_index, body=body, ) @@ -51,15 +52,18 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -69,17 +73,11 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py new file mode 100644 index 0000000000..d10d1ee7bc --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -0,0 +1,37 @@ +from eth2spec.test.context import expect_assertion_error + + +def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): + """ + Run ``process_shard_transitions``, yielding: + - pre-state ('pre') + - shard_transitions ('shard_transitions') + - attestations ('attestations') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # yield pre-state + yield 'pre', state + yield 'shard_transitions', shard_transitions + yield 'attestations', attestations + + # If the attestation is invalid, processing is aborted, and there is no post-state. + if not valid: + expect_assertion_error(lambda: spec.process_shard_transitions(state, shard_transitions, attestations)) + yield 'post', None + return + + # process crosslinks + spec.process_shard_transitions(state, shard_transitions, attestations) + + # yield post-state + yield 'post', state + + +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): + if shard_blocks is None: + shard_blocks = [] + + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) + return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 11ead60337..0635144988 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,5 +1,5 @@ from eth2spec.test.context import ( - PHASE0, PHASE1, + PHASE0, spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases ) from eth2spec.test.helpers.attestations import sign_indexed_attestation @@ -162,10 +162,7 @@ def test_same_data(spec, state): indexed_att_1 = attester_slashing.attestation_1 att_2_data = get_attestation_2_data(spec, attester_slashing) - if spec.fork == PHASE1: - indexed_att_1.attestation.data = att_2_data - else: - indexed_att_1.data = att_2_data + indexed_att_1.data = att_2_data sign_indexed_attestation(spec, state, attester_slashing.attestation_1) yield from run_attester_slashing_processing(spec, state, attester_slashing, False) diff --git a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py index eff2864484..bafefcad63 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_rewards_and_penalties.py @@ -14,6 +14,7 @@ get_valid_attestation, prepare_state_with_attestations, ) +from eth2spec.test.helpers.rewards import leaking from eth2spec.test.helpers.attester_slashings import get_indexed_attestation_participants from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with from random import Random @@ -62,24 +63,6 @@ def test_genesis_epoch_full_attestations_no_rewards(spec, state): assert state.balances[index] == pre_state.balances[index] -@with_all_phases -@spec_state_test -def test_full_attestations(spec, state): - attestations = prepare_state_with_attestations(spec, state) - - pre_state = state.copy() - - yield from run_process_rewards_and_penalties(spec, state) - - attesting_indices = spec.get_unslashed_attesting_indices(state, attestations) - assert len(attesting_indices) == len(pre_state.validators) - for index in range(len(pre_state.validators)): - if index in attesting_indices: - assert state.balances[index] > pre_state.balances[index] - else: - assert state.balances[index] < pre_state.balances[index] - - @with_all_phases @spec_state_test def test_full_attestations_random_incorrect_fields(spec, state): @@ -173,6 +156,7 @@ def participation_tracker(slot, comm_index, comm): return att_participants attestations = prepare_state_with_attestations(spec, state, participation_fn=participation_tracker) + proposer_indices = [a.proposer_index for a in state.previous_epoch_attestations] pre_state = state.copy() @@ -182,10 +166,20 @@ def participation_tracker(slot, comm_index, comm): assert len(attesting_indices) == len(participated) for index in range(len(pre_state.validators)): - if index in participated: - assert state.balances[index] > pre_state.balances[index] + if spec.is_in_inactivity_leak(state): + # Proposers can still make money during a leak + if index in proposer_indices and index in participated: + assert state.balances[index] > pre_state.balances[index] + # If not proposer but participated optimally, should have exactly neutral balance + elif index in attesting_indices: + assert state.balances[index] == pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] else: - assert state.balances[index] < pre_state.balances[index] + if index in participated: + assert state.balances[index] > pre_state.balances[index] + else: + assert state.balances[index] < pre_state.balances[index] @with_all_phases @@ -195,6 +189,14 @@ def test_almost_empty_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) +@with_all_phases +@spec_state_test +@leaking() +def test_almost_empty_attestations_with_leak(spec, state): + rng = Random(1234) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, 1)) + + @with_all_phases @spec_state_test def test_random_fill_attestations(spec, state): @@ -202,6 +204,14 @@ def test_random_fill_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) +@with_all_phases +@spec_state_test +@leaking() +def test_random_fill_attestations_with_leak(spec, state): + rng = Random(4567) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) // 3)) + + @with_all_phases @spec_state_test def test_almost_full_attestations(spec, state): @@ -209,12 +219,27 @@ def test_almost_full_attestations(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) +@with_all_phases +@spec_state_test +@leaking() +def test_almost_full_attestations_with_leak(spec, state): + rng = Random(8901) + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: rng.sample(comm, len(comm) - 1)) + + @with_all_phases @spec_state_test def test_full_attestation_participation(spec, state): yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) +@with_all_phases +@spec_state_test +@leaking() +def test_full_attestation_participation_with_leak(spec, state): + yield from run_with_participation(spec, state, lambda slot, comm_index, comm: comm) + + @with_all_phases @spec_state_test def test_duplicate_attestation(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py index 4e75079c05..b0f9767b23 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/rewards/test_leak.py @@ -1,40 +1,6 @@ from eth2spec.test.context import with_all_phases, spec_state_test -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.rewards import leaking import eth2spec.test.helpers.rewards as rewards_helpers -from lru import LRU - - -def transition_state_to_leak(spec, state, epochs=None): - if epochs is None: - epochs = spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - assert epochs >= spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY - - for _ in range(epochs): - next_epoch(spec, state) - - -_cache_dict = LRU(size=10) - - -def leaking(epochs=None): - - def deco(fn): - def entry(*args, spec, state, **kw): - # If the pre-state is not already known in the LRU, then take it, - # transition it to leak, and put it in the LRU. - # The input state is likely already cached, so the hash-tree-root does not affect speed. - key = (state.hash_tree_root(), spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY, spec.SLOTS_PER_EPOCH, epochs) - global _cache_dict - if key not in _cache_dict: - transition_state_to_leak(spec, state, epochs=epochs) - _cache_dict[key] = state.get_backing() # cache the tree structure, not the view wrapping it. - - # Take an entry out of the LRU. - # No copy is necessary, as we wrap the immutable backing with a new view. - state = spec.BeaconState(backing=_cache_dict[key]) - return fn(*args, spec=spec, state=state, **kw) - return entry - return deco @with_all_phases diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index f0cfc462e6..1e057e5a95 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -16,8 +16,9 @@ get_indexed_attestation_participants, ) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect -from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations +from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, @@ -687,14 +688,23 @@ def test_attestation(spec, state): yield 'pre', state - attestation = get_valid_attestation(spec, state, signed=True, on_time=True) + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + index = 0 + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(state, index, state.slot) + shard_transition = get_shard_transition_of_committee(spec, state, index) + attestation_block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + attestation = get_valid_attestation( + spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True + ) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, state, attestation_block) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py index ed43283274..34ff284129 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_attestation.py @@ -25,22 +25,9 @@ def test_on_time_success(spec, state): @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_on_time_empty_custody_bits_blocks(spec, state): +def test_late_success(spec, state): attestation = get_valid_late_attestation(spec, state, signed=True) - assert not any(attestation.custody_bits_blocks) - - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - yield from run_attestation_processing(spec, state, attestation, False) - - -@with_all_phases_except(['phase0']) -@spec_state_test -@always_bls -def test_late_with_custody_bits_blocks(spec, state): - attestation = get_valid_on_time_attestation(spec, state, signed=True) - transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY + 1) - yield from run_attestation_processing(spec, state, attestation, False) + yield from run_attestation_processing(spec, state, attestation) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py new file mode 100644 index 0000000000..bdb4325fd3 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_chunk_challenge.py @@ -0,0 +1,302 @@ +from eth2spec.test.helpers.custody import ( + get_valid_chunk_challenge, + get_valid_custody_chunk_response, + get_sample_shard_transition, +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.test.helpers.state import transition_to +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing + + +def run_chunk_challenge_processing(spec, state, custody_chunk_challenge, valid=True): + """ + Run ``process_chunk_challenge``, yielding: + - pre-state ('pre') + - CustodyBitChallenge ('custody_chunk_challenge') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_chunk_challenge', custody_chunk_challenge + + if not valid: + expect_assertion_error(lambda: spec.process_chunk_challenge(state, custody_chunk_challenge)) + yield 'post', None + return + + spec.process_chunk_challenge(state, custody_chunk_challenge) + + assert state.custody_chunk_challenge_records[state.custody_chunk_challenge_index - 1].responder_index == \ + custody_chunk_challenge.responder_index + assert state.custody_chunk_challenge_records[state.custody_chunk_challenge_index - 1].chunk_index == \ + custody_chunk_challenge.chunk_index + + yield 'post', state + + +def run_custody_chunk_response_processing(spec, state, custody_response, valid=True): + """ + Run ``process_chunk_challenge_response``, yielding: + - pre-state ('pre') + - CustodyResponse ('custody_response') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_response', custody_response + + if not valid: + expect_assertion_error(lambda: spec.process_custody_response(state, custody_response)) + yield 'post', None + return + + spec.process_chunk_challenge_response(state, custody_response) + + assert state.custody_chunk_challenge_records[custody_response.challenge_index] == spec.CustodyChunkChallengeRecord() + + yield 'post', state + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_challenge_appended(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_challenge_empty_element_replaced(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + state.custody_chunk_challenge_records.append(spec.CustodyChunkChallengeRecord()) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_duplicate_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + yield from run_chunk_challenge_processing(spec, state, challenge, valid=False) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_second_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * spec.EPOCHS_PER_CUSTODY_PERIOD) + + challenge0 = get_valid_chunk_challenge(spec, state, attestation, shard_transition, chunk_index=0) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge0) + + challenge1 = get_valid_chunk_challenge(spec, state, attestation, shard_transition, chunk_index=1) + + yield from run_chunk_challenge_processing(spec, state, challenge1) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_multiple_epochs_custody(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_many_epochs_custody(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_off_chain_attestation(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + yield from run_chunk_challenge_processing(spec, state, challenge) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response_multiple_epochs(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 3) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_response_many_epochs(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * 20) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py index 8c2436d5b6..cb96c97e1c 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_key_reveal.py @@ -28,30 +28,14 @@ def run_custody_key_reveal_processing(spec, state, custody_key_reveal, valid=Tru pre_next_custody_secret_to_reveal = \ state.validators[revealer_index].next_custody_secret_to_reveal - pre_reveal_lateness = state.validators[revealer_index].max_reveal_lateness spec.process_custody_key_reveal(state, custody_key_reveal) post_next_custody_secret_to_reveal = \ state.validators[revealer_index].next_custody_secret_to_reveal - post_reveal_lateness = state.validators[revealer_index].max_reveal_lateness assert post_next_custody_secret_to_reveal == pre_next_custody_secret_to_reveal + 1 - if spec.get_current_epoch(state) > spec.get_randao_epoch_for_custody_period( - pre_next_custody_secret_to_reveal, - revealer_index - ) + spec.EPOCHS_PER_CUSTODY_PERIOD: - assert post_reveal_lateness > 0 - if pre_reveal_lateness == 0: - assert post_reveal_lateness == spec.get_current_epoch(state) - spec.get_randao_epoch_for_custody_period( - pre_next_custody_secret_to_reveal, - revealer_index - ) - spec.EPOCHS_PER_CUSTODY_PERIOD - else: - if pre_reveal_lateness > 0: - assert post_reveal_lateness < pre_reveal_lateness - yield 'post', state @@ -103,17 +87,3 @@ def test_double_reveal(spec, state): _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal, False) - - -@with_all_phases_except([PHASE0]) -@spec_state_test -@always_bls -def test_max_decrement(spec, state): - state.slot += spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH * 3 + 150 - custody_key_reveal = get_valid_custody_key_reveal(spec, state) - - _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) - - custody_key_reveal2 = get_valid_custody_key_reveal(spec, state) - - yield from run_custody_key_reveal_processing(spec, state, custody_key_reveal2) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py new file mode 100644 index 0000000000..ec0bac82d4 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_custody_slashing.py @@ -0,0 +1,137 @@ +from eth2spec.test.helpers.custody import ( + get_valid_custody_slashing, + get_custody_secret, + get_custody_slashable_shard_transition, +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.utils.ssz.ssz_typing import ByteList +from eth2spec.test.helpers.state import get_balance, transition_to +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, + expect_assertion_error, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing + + +def run_custody_slashing_processing(spec, state, custody_slashing, valid=True, correct=True): + """ + Run ``process_bit_challenge``, yielding: + - pre-state ('pre') + - CustodySlashing ('custody_slashing') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'custody_slashing', custody_slashing + + if not valid: + expect_assertion_error(lambda: spec.process_custody_slashing(state, custody_slashing)) + yield 'post', None + return + + if correct: + pre_slashed_balance = get_balance(state, custody_slashing.message.malefactor_index) + else: + pre_slashed_balance = get_balance(state, custody_slashing.message.whistleblower_index) + + spec.process_custody_slashing(state, custody_slashing) + + if correct: + slashed_validator = state.validators[custody_slashing.message.malefactor_index] + assert get_balance(state, custody_slashing.message.malefactor_index) < pre_slashed_balance + else: + slashed_validator = state.validators[custody_slashing.message.whistleblower_index] + assert get_balance(state, custody_slashing.message.whistleblower_index) < pre_slashed_balance + + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + yield 'post', state + + +def run_standard_custody_slashing_test(spec, + state, + shard_lateness=None, + shard=None, + validator_index=None, + block_lengths=None, + slashing_message_data=None, + correct=True, + valid=True): + if shard_lateness is None: + shard_lateness = spec.SLOTS_PER_EPOCH + transition_to(spec, state, state.slot + shard_lateness) + + if shard is None: + shard = 0 + if validator_index is None: + validator_index = spec.get_beacon_committee(state, state.slot, shard)[0] + + offset_slots = spec.get_offset_slots(state, shard) + if block_lengths is None: + block_lengths = [2**15 // 3] * len(offset_slots) + + custody_secret = get_custody_secret(spec, state, validator_index) + shard_transition, slashable_test_vector = get_custody_slashable_shard_transition( + spec, + state.slot, + block_lengths, + custody_secret, + slashable=correct, + ) + + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + slashing = get_valid_custody_slashing(spec, state, attestation, shard_transition, + custody_secret, slashable_test_vector) + + if slashing_message_data is not None: + slashing.message.data = slashing_message_data + + yield from run_custody_slashing_processing(spec, state, slashing, valid=valid, correct=correct) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_custody_slashing(spec, state): + yield from run_standard_custody_slashing_test(spec, state) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_incorrect_custody_slashing(spec, state): + yield from run_standard_custody_slashing_test(spec, state, correct=False) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_multiple_epochs_custody(spec, state): + yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 3) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_many_epochs_custody(spec, state): + yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 10) + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_invalid_custody_slashing(spec, state): + yield from run_standard_custody_slashing_test( + spec, + state, + slashing_message_data=ByteList[spec.MAX_SHARD_BLOCK_SIZE](), + valid=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py similarity index 61% rename from tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py rename to tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 1f066b3442..6d88781772 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,72 +2,70 @@ PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) -from eth2spec.test.helpers.crosslinks import run_crosslinks_processing +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation +from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - assert state.shard_states[shard].slot == slot_x - 1 + init_slot = state.slot + shard_slot = state.slot + target_len_offset_slot - 1 + shard = spec.compute_shard_from_committee_index(state, committee_index, shard_slot) + assert state.shard_states[shard].slot == init_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] + + # Transition state latest shard slot + transition_to(spec, state, shard_slot) # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - transition_to(spec, state, state.slot + target_len_offset_slot) + transition_to(spec, state, init_slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - - yield from run_crosslinks_processing(spec, state, shard_transitions, [attestation], valid=valid) + yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == slot_x + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py new file mode 100644 index 0000000000..f3675732a4 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_challenge_deadlines.py @@ -0,0 +1,57 @@ +from eth2spec.test.helpers.custody import ( + get_valid_chunk_challenge, + get_sample_shard_transition, +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.test.helpers.state import transition_to +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with + +from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import ( + run_chunk_challenge_processing, +) + + +def run_process_challenge_deadlines(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_slashed_after_chunk_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index + )[0] + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + assert state.validators[validator_index].slashed == 0 + + transition_to(spec, state, state.slot + spec.MAX_CHUNK_CHALLENGE_DELAY * spec.SLOTS_PER_EPOCH) + + state.validators[validator_index].slashed = 0 + + yield from run_process_challenge_deadlines(spec, state) + + assert state.validators[validator_index].slashed == 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py new file mode 100644 index 0000000000..6ca6c8c99f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_custody_final_updates.py @@ -0,0 +1,165 @@ +from eth2spec.test.helpers.custody import ( + get_valid_chunk_challenge, + get_valid_custody_chunk_response, + get_valid_custody_key_reveal, + get_sample_shard_transition +) +from eth2spec.test.helpers.attestations import ( + get_valid_on_time_attestation, +) +from eth2spec.test.helpers.state import next_epoch_via_block, transition_to +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.phase_0.block_processing.test_process_attestation import run_attestation_processing +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with + +from eth2spec.test.phase_1.block_processing.test_process_chunk_challenge import ( + run_chunk_challenge_processing, + run_custody_chunk_response_processing, +) +from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing + + +def run_process_custody_final_updates(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_custody_final_updates') + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_delay(spec, state): + spec.initiate_validator_exit(state, 0) + assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + yield from run_process_custody_final_updates(spec, state) + + assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_reenable_after_custody_reveal(spec, state): + spec.initiate_validator_exit(state, 0) + assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch_via_block(spec, state) + + assert state.validators[0].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[0].exit_epoch: + next_epoch_via_block(spec, state) + + while (state.validators[0].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator(0, state.validators[0].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=0) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + yield from run_process_custody_final_updates(spec, state) + + assert state.validators[0].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_suspend_after_chunk_challenge(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch_via_block(spec, state) + + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch_via_block(spec, state) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + yield from run_process_custody_final_updates(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_withdrawal_resume_after_chunk_challenge_response(spec, state): + transition_to(spec, state, state.slot + 1) + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + validator_index = spec.get_beacon_committee( + state, + attestation.data.slot, + attestation.data.index + )[0] + + spec.initiate_validator_exit(state, validator_index) + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH + + next_epoch_via_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + while spec.get_current_epoch(state) < state.validators[validator_index].exit_epoch: + next_epoch_via_block(spec, state) + + while (state.validators[validator_index].next_custody_secret_to_reveal + <= spec.get_custody_period_for_validator( + validator_index, + state.validators[validator_index].exit_epoch - 1)): + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=validator_index) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + next_epoch_via_block(spec, state) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + next_epoch_via_block(spec, state) + + assert state.validators[validator_index].withdrawable_epoch == spec.FAR_FUTURE_EPOCH + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + custody_response = get_valid_custody_chunk_response(spec, state, challenge, 2**15 // 3, chunk_challenge_index) + + _, _, _ = run_custody_chunk_response_processing(spec, state, custody_response) + + yield from run_process_custody_final_updates(spec, state) + + assert state.validators[validator_index].withdrawable_epoch < spec.FAR_FUTURE_EPOCH diff --git a/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py new file mode 100644 index 0000000000..9cc0069b93 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/epoch_processing/test_process_reveal_deadlines.py @@ -0,0 +1,50 @@ +from eth2spec.test.helpers.custody import ( + get_valid_custody_key_reveal, +) +from eth2spec.test.helpers.state import transition_to +from eth2spec.test.context import ( + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import run_epoch_processing_with +from eth2spec.test.phase_1.block_processing.test_process_custody_key_reveal import run_custody_key_reveal_processing + + +def run_process_challenge_deadlines(spec, state): + yield from run_epoch_processing_with(spec, state, 'process_challenge_deadlines') + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_slashed_after_reveal_deadline(spec, state): + assert state.validators[0].slashed == 0 + transition_to(spec, state, spec.get_randao_epoch_for_custody_period(0, 0) * spec.SLOTS_PER_EPOCH) + + # Need to run at least one reveal so that not all validators are slashed (otherwise spec fails to find proposers) + custody_key_reveal = get_valid_custody_key_reveal(spec, state, validator_index=1) + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) + + state.validators[0].slashed = 0 + + yield from run_process_challenge_deadlines(spec, state) + + assert state.validators[0].slashed == 1 + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_validator_not_slashed_after_reveal(spec, state): + transition_to(spec, state, spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) + custody_key_reveal = get_valid_custody_key_reveal(spec, state) + + _, _, _ = run_custody_key_reveal_processing(spec, state, custody_key_reveal) + + assert state.validators[0].slashed == 0 + + transition_to(spec, state, state.slot + spec.EPOCHS_PER_CUSTODY_PERIOD * spec.SLOTS_PER_EPOCH) + + yield from run_process_challenge_deadlines(spec, state) + + assert state.validators[0].slashed == 0 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 60af35d45d..33b0beac7b 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,56 +55,37 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py new file mode 100644 index 0000000000..27afd4a4e8 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -0,0 +1,71 @@ +from eth2spec.test.context import ( + PHASE0, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import next_epoch + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_committee_count_delta(spec, state): + assert spec.get_committee_count_delta(state, 0, 0) == 0 + assert spec.get_committee_count_at_slot(state, 0) != 0 + assert spec.get_committee_count_delta(state, 0, 1) == spec.get_committee_count_at_slot(state, 0) + assert spec.get_committee_count_delta(state, 1, 2) == spec.get_committee_count_at_slot(state, 1) + assert spec.get_committee_count_delta(state, 0, 2) == ( + spec.get_committee_count_at_slot(state, 0) + spec.get_committee_count_at_slot(state, 1) + ) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_current_epoch_start(spec, state): + assert state.current_epoch_start_shard == 0 + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + assert state.current_epoch_start_shard == ( + spec.get_committee_count_delta(state, 0, spec.SLOTS_PER_EPOCH) % active_shard_count + ) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + start_shard = spec.get_start_shard(state, slot) + assert start_shard == state.current_epoch_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_next_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + ) % active_shard_count + assert start_shard == expected_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_previous_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot - 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH * active_shard_count + - spec.get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + ) % active_shard_count + assert start_shard == expected_start_shard diff --git a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py index 5bb246ed5c..e749b1e3d0 100644 --- a/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py @@ -1,5 +1,9 @@ -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -from eth2spec.test.helpers.attestations import build_attestation_data +from eth2spec.test.context import ( + spec_state_test, + always_bls, with_phases, with_all_phases, + PHASE0, +) +from eth2spec.test.helpers.attestations import build_attestation_data, get_valid_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.deposits import prepare_state_and_deposit from eth2spec.test.helpers.keys import privkeys, pubkeys @@ -8,9 +12,11 @@ from eth2spec.utils.ssz.ssz_typing import Bitlist -def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey): +def run_get_signature_test(spec, state, obj, domain, get_signature_fn, privkey, pubkey, signing_ssz_object=None): + if signing_ssz_object is None: + signing_ssz_object = obj signature = get_signature_fn(state, obj, privkey) - signing_root = spec.compute_signing_root(obj, domain) + signing_root = spec.compute_signing_root(signing_ssz_object, domain) assert bls.Verify(pubkey, signing_root, signature) @@ -55,7 +61,6 @@ def get_mock_aggregate(spec): @with_all_phases @spec_state_test -@never_bls def test_check_if_validator_active(spec, state): active_validator_index = len(state.validators) - 1 assert spec.check_if_validator_active(state, active_validator_index) @@ -73,7 +78,6 @@ def test_check_if_validator_active(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_current_epoch(spec, state): epoch = spec.get_current_epoch(state) validator_index = len(state.validators) - 1 @@ -82,7 +86,6 @@ def test_get_committee_assignment_current_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_next_epoch(spec, state): epoch = spec.get_current_epoch(state) + 1 validator_index = len(state.validators) - 1 @@ -91,7 +94,6 @@ def test_get_committee_assignment_next_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_get_committee_assignment_out_bound_epoch(spec, state): epoch = spec.get_current_epoch(state) + 2 validator_index = len(state.validators) - 1 @@ -100,7 +102,6 @@ def test_get_committee_assignment_out_bound_epoch(spec, state): @with_all_phases @spec_state_test -@never_bls def test_is_proposer(spec, state): proposer_index = spec.get_beacon_proposer_index(state) assert spec.is_proposer(state, proposer_index) @@ -132,6 +133,7 @@ def test_get_epoch_signature(spec, state): get_signature_fn=spec.get_epoch_signature, privkey=privkey, pubkey=pubkey, + signing_ssz_object=spec.compute_epoch_at_slot(block.slot), ) @@ -190,8 +192,16 @@ def test_get_eth1_vote_consensus_vote(spec, state): assert votes_length >= 3 # We need to have the majority vote state.eth1_data_votes = () - block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) - block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, + deposit_count=state.eth1_data.deposit_count, + deposit_root=b'\x04' * 32, + ) + block_2 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count + 1, + deposit_root=b'\x05' * 32, + ) eth1_chain = [block_1, block_2] eth1_data_votes = [] @@ -218,8 +228,16 @@ def test_get_eth1_vote_tie(spec, state): assert votes_length > 0 and votes_length % 2 == 0 state.eth1_data_votes = () - block_1 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1) - block_2 = spec.Eth1Block(timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE) + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE - 1, + deposit_count=state.eth1_data.deposit_count, + deposit_root=b'\x04' * 32, + ) + block_2 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count + 1, + deposit_root=b'\x05' * 32, + ) eth1_chain = [block_1, block_2] eth1_data_votes = [] # Half votes are for block_1, another half votes are for block_2 @@ -237,6 +255,33 @@ def test_get_eth1_vote_tie(spec, state): assert eth1_data.block_hash == eth1_chain[0].hash_tree_root() +@with_all_phases +@spec_state_test +def test_get_eth1_vote_chain_in_past(spec, state): + min_new_period_epochs = get_min_new_period_epochs(spec) + for _ in range(min_new_period_epochs + 1): + next_epoch(spec, state) + + period_start = spec.voting_period_start_time(state) + votes_length = spec.get_current_epoch(state) % spec.EPOCHS_PER_ETH1_VOTING_PERIOD + assert votes_length > 0 and votes_length % 2 == 0 + + state.eth1_data_votes = () + block_1 = spec.Eth1Block( + timestamp=period_start - spec.SECONDS_PER_ETH1_BLOCK * spec.ETH1_FOLLOW_DISTANCE, + deposit_count=state.eth1_data.deposit_count - 1, # Chain prior to current eth1data + deposit_root=b'\x42' * 32, + ) + eth1_chain = [block_1] + eth1_data_votes = [] + + state.eth1_data_votes = eth1_data_votes + eth1_data = spec.get_eth1_vote(state, eth1_chain) + + # Should be default vote + assert eth1_data == state.eth1_data + + @with_all_phases @spec_state_test def test_compute_new_state_root(spec, state): @@ -256,6 +301,7 @@ def test_compute_new_state_root(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_block_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -272,20 +318,35 @@ def test_get_block_signature(spec, state): ) +@with_all_phases +@spec_state_test +def test_compute_fork_digest(spec, state): + actual_fork_digest = spec.compute_fork_digest(state.fork.current_version, state.genesis_validators_root) + + expected_fork_data_root = spec.hash_tree_root( + spec.ForkData(current_version=state.fork.current_version, + genesis_validators_root=state.genesis_validators_root)) + expected_fork_digest = spec.ForkDigest(expected_fork_data_root[:4]) + + assert actual_fork_digest == expected_fork_digest + + # Attesting @with_all_phases @spec_state_test -def test_get_attestation_signature(spec, state): +@always_bls +def test_get_attestation_signature_phase0(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] - attestation_data = spec.AttestationData(slot=10) - domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + attestation = get_valid_attestation(spec, state, signed=False) + domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch) + run_get_signature_test( spec=spec, state=state, - obj=attestation_data, + obj=attestation.data, domain=domain, get_signature_fn=spec.get_attestation_signature, privkey=privkey, @@ -293,11 +354,28 @@ def test_get_attestation_signature(spec, state): ) +@with_all_phases +@spec_state_test +def test_compute_subnet_for_attestation(spec, state): + for committee_idx in range(spec.MAX_COMMITTEES_PER_SLOT): + for slot in range(state.slot, state.slot + spec.SLOTS_PER_EPOCH): + actual_subnet_id = spec.compute_subnet_for_attestation(state, slot, committee_idx) + + slots_since_epoch_start = slot % spec.SLOTS_PER_EPOCH + committees_since_epoch_start = spec.get_committee_count_at_slot( + state, slot) * slots_since_epoch_start + expected_subnet_id = (committees_since_epoch_start + + committee_idx) % spec.ATTESTATION_SUBNET_COUNT + + assert actual_subnet_id == expected_subnet_id + + # Attestation aggregation @with_all_phases @spec_state_test +@always_bls def test_get_slot_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] @@ -316,9 +394,10 @@ def test_get_slot_signature(spec, state): @with_all_phases @spec_state_test +@always_bls def test_is_aggregator(spec, state): # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE` - # if we have more validators and larger committeee size + # if we have more validators and larger committee size slot = state.slot committee_index = 0 has_aggregator = False @@ -332,11 +411,12 @@ def test_is_aggregator(spec, state): assert has_aggregator -@with_all_phases +@with_phases([PHASE0]) @spec_state_test +@always_bls def test_get_aggregate_signature(spec, state): attestations = [] - pubkeys = [] + attesting_pubkeys = [] slot = state.slot committee_index = 0 attestation_data = build_attestation_data(spec, state, slot=slot, index=committee_index) @@ -348,24 +428,26 @@ def test_get_aggregate_signature(spec, state): committee_size = len(beacon_committee) aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) for i, validator_index in enumerate(beacon_committee): - bits = aggregation_bits + bits = aggregation_bits.copy() bits[i] = True attestations.append( spec.Attestation( data=attestation_data, aggregation_bits=bits, + signature=spec.get_attestation_signature(state, attestation_data, privkeys[validator_index]), ) ) - pubkeys.append(state.validators[validator_index].pubkey) - pubkey = bls.AggregatePKs(pubkeys) + attesting_pubkeys.append(state.validators[validator_index].pubkey) + assert len(attestations) > 0 signature = spec.get_aggregate_signature(attestations) domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) signing_root = spec.compute_signing_root(attestation_data, domain) - assert bls.Verify(pubkey, signing_root, signature) + assert bls.FastAggregateVerify(attesting_pubkeys, signing_root, signature) @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_and_proof(spec, state): privkey = privkeys[0] aggregator_index = spec.ValidatorIndex(10) @@ -378,6 +460,7 @@ def test_get_aggregate_and_proof(spec, state): @with_all_phases @spec_state_test +@always_bls def test_get_aggregate_and_proof_signature(spec, state): privkey = privkeys[0] pubkey = pubkeys[0] diff --git a/tests/core/pyspec/eth2spec/utils/bls.py b/tests/core/pyspec/eth2spec/utils/bls.py index acf9f99c7d..778b23da7e 100644 --- a/tests/core/pyspec/eth2spec/utils/bls.py +++ b/tests/core/pyspec/eth2spec/utils/bls.py @@ -1,12 +1,17 @@ -from py_ecc.bls import G2ProofOfPossession as bls +from py_ecc.bls import G2ProofOfPossession as py_ecc_bls from py_ecc.bls.g2_primatives import signature_to_G2 as _signature_to_G2 +import milagro_bls_binding as milagro_bls # noqa: F401 for BLS switching option # Flag to make BLS active or not. Used for testing, do not ignore BLS in production unless you know what you are doing. bls_active = True +# To change bls implementation, default to PyECC for correctness. Milagro is a good faster alternative. +bls = py_ecc_bls + STUB_SIGNATURE = b'\x11' * 96 STUB_PUBKEY = b'\x22' * 48 -STUB_COORDINATES = _signature_to_G2(bls.Sign(0, b"")) +Z2_SIGNATURE = b'\xc0' + b'\x00' * 95 +STUB_COORDINATES = _signature_to_G2(Z2_SIGNATURE) def only_with_bls(alt_return=None): @@ -36,7 +41,7 @@ def Verify(PK, message, signature): @only_with_bls(alt_return=True) def AggregateVerify(pubkeys, messages, signature): try: - result = bls.AggregateVerify(pubkeys, messages, signature) + result = bls.AggregateVerify(list(pubkeys), list(messages), signature) except Exception: result = False finally: @@ -46,7 +51,7 @@ def AggregateVerify(pubkeys, messages, signature): @only_with_bls(alt_return=True) def FastAggregateVerify(pubkeys, message, signature): try: - result = bls.FastAggregateVerify(pubkeys, message, signature) + result = bls.FastAggregateVerify(list(pubkeys), message, signature) except Exception: result = False finally: @@ -60,7 +65,10 @@ def Aggregate(signatures): @only_with_bls(alt_return=STUB_SIGNATURE) def Sign(SK, message): - return bls.Sign(SK, message) + if bls == py_ecc_bls: + return bls.Sign(SK, message) + else: + return bls.Sign(SK.to_bytes(32, 'big'), message) @only_with_bls(alt_return=STUB_COORDINATES) @@ -70,7 +78,7 @@ def signature_to_G2(signature): @only_with_bls(alt_return=STUB_PUBKEY) def AggregatePKs(pubkeys): - return bls._AggregatePKs(pubkeys) + return bls._AggregatePKs(list(pubkeys)) @only_with_bls(alt_return=STUB_SIGNATURE) diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index 34e6c4ee8c..ad5a0fedf4 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,3 +1,4 @@ +from typing import TypeVar from remerkleable.core import View from remerkleable.byte_arrays import Bytes32 @@ -8,3 +9,11 @@ def serialize(obj: View) -> bytes: def hash_tree_root(obj: View) -> Bytes32: return Bytes32(obj.get_backing().merkle_root()) + + +V = TypeVar('V', bound=View) + + +# Helper method for typing copies, and avoiding a example_input.copy() method call, instead of copy(example_input) +def copy(obj: V) -> V: + return obj.copy() diff --git a/tests/generators/README.md b/tests/generators/README.md index 77a50606b5..9446551fb7 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -184,7 +184,7 @@ def create_provider(handler_name: str, tests_src, config_name: str) -> gen_typin if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_provider('crosslinks', test_process_crosslinks, 'minimal'), + create_provider('final_updates', test_process_final_updates, 'minimal'), ... ]) diff --git a/tests/generators/bls/main.py b/tests/generators/bls/main.py index 8c6589b364..9e10b4044b 100644 --- a/tests/generators/bls/main.py +++ b/tests/generators/bls/main.py @@ -2,18 +2,22 @@ BLS test vectors generator """ +from hashlib import sha256 from typing import Tuple, Iterable, Any, Callable, Dict from eth_utils import ( encode_hex, int_to_big_endian, ) -from gen_base import gen_runner, gen_typing +import milagro_bls_binding as milagro_bls from eth2spec.utils import bls -from hashlib import sha256 - from eth2spec.test.context import PHASE0 +from gen_base import gen_runner, gen_typing + + +def to_bytes(i): + return i.to_bytes(32, "big") def hash(x): @@ -70,8 +74,15 @@ def case02_verify(): # Valid signature signature = bls.Sign(privkey, message) pubkey = bls.SkToPk(privkey) + + assert milagro_bls.SkToPk(to_bytes(privkey)) == pubkey + assert milagro_bls.Sign(to_bytes(privkey), message) == signature + identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' + assert bls.Verify(pubkey, message, signature) + assert milagro_bls.Verify(pubkey, message, signature) + yield f'verify_valid_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -85,6 +96,7 @@ def case02_verify(): wrong_pubkey = bls.SkToPk(PRIVKEYS[(i + 1) % len(PRIVKEYS)]) identifier = f'{encode_hex(wrong_pubkey)}_{encode_hex(message)}' assert not bls.Verify(wrong_pubkey, message, signature) + assert not milagro_bls.Verify(wrong_pubkey, message, signature) yield f'verify_wrong_pubkey_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(wrong_pubkey), @@ -98,6 +110,7 @@ def case02_verify(): tampered_signature = signature[:-4] + b'\xFF\xFF\xFF\xFF' identifier = f'{encode_hex(pubkey)}_{encode_hex(message)}' assert not bls.Verify(pubkey, message, tampered_signature) + assert not milagro_bls.Verify(pubkey, message, tampered_signature) yield f'verify_tampered_signature_case_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkey': encode_hex(pubkey), @@ -109,6 +122,7 @@ def case02_verify(): # Valid pubkey and signature with the point at infinity assert bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) + assert milagro_bls.Verify(Z1_PUBKEY, message, Z2_SIGNATURE) yield f'verify_infinity_pubkey_and_infinity_signature', { 'input': { 'pubkey': encode_hex(Z1_PUBKEY), @@ -152,6 +166,7 @@ def case04_fast_aggregate_verify(): # Valid signature identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert bls.FastAggregateVerify(pubkeys, message, aggregate_signature) + assert milagro_bls.FastAggregateVerify(pubkeys, message, aggregate_signature) yield f'fast_aggregate_verify_valid_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -166,6 +181,7 @@ def case04_fast_aggregate_verify(): pubkeys_extra_serial = [encode_hex(pubkey) for pubkey in pubkeys_extra] identifier = f'{pubkeys_extra_serial}_{encode_hex(message)}' assert not bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) + assert not milagro_bls.FastAggregateVerify(pubkeys_extra, message, aggregate_signature) yield f'fast_aggregate_verify_extra_pubkey_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_extra_serial, @@ -179,6 +195,7 @@ def case04_fast_aggregate_verify(): tampered_signature = aggregate_signature[:-4] + b'\xff\xff\xff\xff' identifier = f'{pubkeys_serial}_{encode_hex(message)}' assert not bls.FastAggregateVerify(pubkeys, message, tampered_signature) + assert not milagro_bls.FastAggregateVerify(pubkeys, message, tampered_signature) yield f'fast_aggregate_verify_tampered_signature_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'pubkeys': pubkeys_serial, @@ -190,6 +207,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.FastAggregateVerify([], message, Z2_SIGNATURE) + assert not milagro_bls.FastAggregateVerify([], message, Z2_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], @@ -201,6 +219,7 @@ def case04_fast_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.FastAggregateVerify([], message, NO_SIGNATURE) + assert not milagro_bls.FastAggregateVerify([], message, NO_SIGNATURE) yield f'fast_aggregate_verify_na_pubkeys_and_na_signature', { 'input': { 'pubkeys': [], @@ -228,6 +247,7 @@ def case05_aggregate_verify(): aggregate_signature = bls.Aggregate(sigs) assert bls.AggregateVerify(pubkeys, messages, aggregate_signature) + assert milagro_bls.AggregateVerify(pubkeys, messages, aggregate_signature) yield f'aggregate_verify_valid', { 'input': { 'pubkeys': pubkeys_serial, @@ -239,6 +259,7 @@ def case05_aggregate_verify(): tampered_signature = aggregate_signature[:4] + b'\xff\xff\xff\xff' assert not bls.AggregateVerify(pubkey, messages, tampered_signature) + assert not milagro_bls.AggregateVerify(pubkeys, messages, tampered_signature) yield f'aggregate_verify_tampered_signature', { 'input': { 'pubkeys': pubkeys_serial, @@ -250,6 +271,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == Z1_SIGNATURE assert not bls.AggregateVerify([], [], Z2_SIGNATURE) + assert not milagro_bls.AggregateVerify([], [], Z2_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_infinity_signature', { 'input': { 'pubkeys': [], @@ -261,6 +283,7 @@ def case05_aggregate_verify(): # Invalid pubkeys and signature -- len(pubkeys) == 0 and signature == 0x00... assert not bls.AggregateVerify([], [], NO_SIGNATURE) + assert not milagro_bls.AggregateVerify([], [], NO_SIGNATURE) yield f'aggregate_verify_na_pubkeys_and_na_signature', { 'input': { 'pubkeys': [],