diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ac281dea9..17db49b3a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,19 +44,6 @@ commands: venv_name: v25-pyspec reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }} venv_path: ./venv - restore_deposit_contract_tester_cached_venv: - description: "Restore the venv from cache for the deposit contract tester" - steps: - - restore_cached_venv: - venv_name: v23-deposit-contract-tester - reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}-{{ checksum "solidity_deposit_contract/web3_tester/requirements.txt" }} - save_deposit_contract_tester_cached_venv: - description: "Save the venv to cache for later use of the deposit contract tester" - steps: - - save_cached_venv: - venv_name: v23-deposit-contract-tester - reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}-{{ checksum "solidity_deposit_contract/web3_tester/requirements.txt" }} - venv_path: ./solidity_deposit_contract/web3_tester/venv jobs: checkout_specs: docker: @@ -226,71 +213,6 @@ jobs: - run: name: Run linter for test generators command: make lint_generators - build_deposit_contract: - docker: - - image: ethereum/solc:0.6.11-alpine - steps: - - checkout - - run: - name: Install build essentials - command: | - apk update - apk add git make - - run: - name: Compile the contract - command: | - make compile_deposit_contract - git diff --color --exit-code - - persist_to_workspace: - root: . - paths: - - ./solidity_deposit_contract/deposit_contract.json - - ./build/combined.json - - ./solidity_deposit_contract/lib - test_deposit_contract: - docker: - - image: nixorg/nix:circleci - steps: - - checkout - - restore_cache: - key: nix-store-test-v2 - - attach_workspace: - at: /tmp/ - - run: - name: Test the contract - command: | - mkdir build - cp -r /tmp/build/* build - cp -r /tmp/solidity_deposit_contract/lib/* ./solidity_deposit_contract/lib - cp -r /tmp/solidity_deposit_contract/deposit_contract.json ./solidity_deposit_contract/deposit_contract.json - nix-shell --command 'make test_deposit_contract' ./solidity_deposit_contract/shell.nix - - save_cache: - key: nix-store-test-v2 - paths: - - /nix - install_deposit_contract_web3_tester: - docker: - - image: cimg/python:3.12.4 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_deposit_contract_tester_cached_venv - - run: - name: Install deposit contract tester requirements - command: make install_deposit_contract_web3_tester - - save_deposit_contract_tester_cached_venv - test_deposit_contract_web3_tests: - docker: - - image: cimg/python:3.12.4 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_deposit_contract_tester_cached_venv - - run: - name: Run deposit contract test with web3.py - command: make test_deposit_contract_web3_tests workflows: version: 2.1 test_spec: @@ -328,17 +250,3 @@ workflows: - lint: requires: - install_pyspec_test - # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. - # - install_deposit_contract_web3_tester: - # requires: - # - checkout_specs - # - test_deposit_contract_web3_tests: - # requires: - # - install_deposit_contract_web3_tester - build_and_test_deposit_contract: - jobs: - - build_deposit_contract - # NOTE: Since phase 0 has been launched, we disabled the deposit contract tests. - # - test_deposit_contract: - # requires: - # - build_deposit_contract diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 83926c47c5..b8c2fa6e14 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -110,3 +110,18 @@ jobs: with: name: test-reports-${{ matrix.version }} path: tests/core/pyspec/test-reports + + gen-modcheck: + runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12.4' + cache: '' + - name: Install pyspec requirements + run: make install_test + - name: Run generators with --modcheck + run: make generate_tests modcheck=true diff --git a/Makefile b/Makefile index b0f7f34ad6..7a3ebbf6c6 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,6 @@ ETH2SPEC_MODULE_DIR = $(PY_SPEC_DIR)/eth2spec TEST_REPORT_DIR = $(PY_SPEC_DIR)/test-reports TEST_VECTOR_DIR = ../consensus-spec-tests/tests GENERATOR_DIR = ./tests/generators -SOLIDITY_DEPOSIT_CONTRACT_DIR = ./solidity_deposit_contract -SOLIDITY_DEPOSIT_CONTRACT_SOURCE = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/deposit_contract.sol -SOLIDITY_FILE_NAME = deposit_contract.json -DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester CONFIGS_DIR = ./configs TEST_PRESET_TYPE ?= minimal # Collect a list of generator names @@ -45,20 +41,12 @@ COV_HTML_OUT_DIR=$(PY_SPEC_DIR)/$(COV_HTML_OUT) COV_INDEX_FILE=$(COV_HTML_OUT_DIR)/index.html CURRENT_DIR = ${CURDIR} -LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt SCRIPTS_DIR = ${CURRENT_DIR}/scripts -export DAPP_SKIP_BUILD:=1 -export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR) -export DAPP_LIB:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)/lib -export DAPP_JSON:=build/combined.json - .PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \ - install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \ - compile_deposit_contract test_compile_deposit_contract check_toc \ - detect_generator_incomplete detect_generator_error_log + check_toc detect_generator_incomplete detect_generator_error_log all: $(PY_SPEC_ALL_TARGETS) @@ -162,33 +150,16 @@ codespell: lint: pyspec . venv/bin/activate; cd $(PY_SPEC_DIR); \ - flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \ - && python -m pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \ - && python -m mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE) + flake8 --config $(CURRENT_DIR)/flake8.ini ./eth2spec \ + && python -m pylint --rcfile $(CURRENT_DIR)/pylint.ini $(PYLINT_SCOPE) \ + && python -m mypy --config-file $(CURRENT_DIR)/mypy.ini $(MYPY_SCOPE) lint_generators: pyspec . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \ - flake8 --config $(LINTER_CONFIG_FILE) - -compile_deposit_contract: - @cd $(SOLIDITY_DEPOSIT_CONTRACT_DIR) - @git submodule update --recursive --init - @solc --metadata-literal --optimize --optimize-runs 5000000 --bin --abi --combined-json=abi,bin,bin-runtime,srcmap,srcmap-runtime,ast,metadata,storage-layout --overwrite -o build $(SOLIDITY_DEPOSIT_CONTRACT_SOURCE) $(SOLIDITY_DEPOSIT_CONTRACT_DIR)/tests/deposit_contract.t.sol - @/bin/echo -n '{"abi": ' > $(SOLIDITY_FILE_NAME) - @cat build/DepositContract.abi >> $(SOLIDITY_FILE_NAME) - @/bin/echo -n ', "bytecode": "0x' >> $(SOLIDITY_FILE_NAME) - @cat build/DepositContract.bin >> $(SOLIDITY_FILE_NAME) - @/bin/echo -n '"}' >> $(SOLIDITY_FILE_NAME) - -test_deposit_contract: - dapp test -v --fuzz-runs 5 - -install_deposit_contract_web3_tester: - cd $(DEPOSIT_CONTRACT_TESTER_DIR); python3 -m venv venv; . venv/bin/activate; python3 -m pip install -r requirements.txt + flake8 --config $(CURRENT_DIR)/flake8.ini -test_deposit_contract_web3_tests: - cd $(DEPOSIT_CONTRACT_TESTER_DIR); . venv/bin/activate; \ - python3 -m pytest . +# If set to true, it will not run generator tests. +modcheck ?= false # Runs a generator, identified by param 1 define run_generator @@ -208,7 +179,7 @@ define run_generator . venv/bin/activate; \ pip3 install ../../../dist/eth2spec-*.whl; \ pip3 install 'eth2spec[generator]'; \ - python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR); \ + python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) $(if $(filter true,$(modcheck)),--modcheck); \ echo "generator $(1) finished" endef diff --git a/README.md b/README.md index fa41dc6298..c3a57c6256 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository hosts the current Ethereum proof-of-stake specifications. Discus [![GitHub release](https://img.shields.io/github/v/release/ethereum/consensus-specs)](https://github.com/ethereum/consensus-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) [![testgen](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml/badge.svg?branch=dev&event=schedule)](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml) -Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features. +Core specifications for Ethereum proof-of-stake clients can be found in [specs](./specs). These are divided into features. Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready. ### Stable Specifications @@ -20,16 +20,22 @@ Features are researched and developed in parallel, and then consolidated into se | 0 | **Phase0** |`0` | | | 1 | **Altair** | `74240` | | | 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | `144896` | | -| 3 | **Capella** | `194048` | | -| 4 | **Deneb** | `269568` | | +| 3 | **Capella** | `194048` | | +| 4 | **Deneb** | `269568` | | ### In-development Specifications + +| Seq. | Code Name | Fork Epoch | Specs | +| - | - | - | - | +| 5 | **Electra** | TBD | | + +### Outdated Specifications + | Code Name or Topic | Specs | Notes | | - | - | - | -| Electra | | -| Sharding (outdated) | | -| Custody Game (outdated) | | Dependent on sharding | -| Data Availability Sampling (outdated) | | | +| Sharding | | +| Custody Game | | Dependent on sharding | +| Data Availability Sampling | | | ### Accompanying documents can be found in [specs](specs) and include: diff --git a/flake8.ini b/flake8.ini new file mode 100644 index 0000000000..f94b2ad47a --- /dev/null +++ b/flake8.ini @@ -0,0 +1,3 @@ +[flake8] +ignore = E252,W504,W503 +max-line-length = 120 \ No newline at end of file diff --git a/linter.ini b/linter.ini deleted file mode 100644 index 52a3aec0e0..0000000000 --- a/linter.ini +++ /dev/null @@ -1,18 +0,0 @@ -[flake8] -ignore = E252,W504,W503 -max-line-length = 120 - -[mypy] -disallow_incomplete_defs = True -disallow_untyped_defs = True - -warn_unused_ignores = True -warn_unused_configs = True -warn_redundant_casts = True - -ignore_missing_imports = True - -# pylint -[MESSAGES CONTROL] -disable = all -enable = unused-argument diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000000..6f16a21c7a --- /dev/null +++ b/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +disallow_incomplete_defs = True +disallow_untyped_defs = True +warn_unused_ignores = True +warn_unused_configs = True +warn_redundant_casts = True +ignore_missing_imports = True \ No newline at end of file diff --git a/pylint.ini b/pylint.ini new file mode 100644 index 0000000000..a8242683a3 --- /dev/null +++ b/pylint.ini @@ -0,0 +1,3 @@ +[MESSAGES CONTROL] +disable = all +enable = unused-argument \ No newline at end of file diff --git a/pysetup/md_doc_paths.py b/pysetup/md_doc_paths.py index d99fc122ac..0f0d1c8593 100644 --- a/pysetup/md_doc_paths.py +++ b/pysetup/md_doc_paths.py @@ -22,7 +22,7 @@ DENEB: CAPELLA, ELECTRA: DENEB, WHISK: CAPELLA, - EIP7594: DENEB, + EIP7594: ELECTRA, EIP6800: DENEB, EIP7732: ELECTRA, } diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index 8985b7f6f4..e3177da8ca 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -10,7 +10,7 @@ class EIP7594SpecBuilder(BaseSpecBuilder): @classmethod def imports(cls, preset_name: str): return f''' -from eth2spec.deneb import {preset_name} as deneb +from eth2spec.electra import {preset_name} as electra ''' diff --git a/pysetup/spec_builders/electra.py b/pysetup/spec_builders/electra.py index 48082249b6..6746d9aada 100644 --- a/pysetup/spec_builders/electra.py +++ b/pysetup/spec_builders/electra.py @@ -10,7 +10,7 @@ class ElectraSpecBuilder(BaseSpecBuilder): def imports(cls, preset_name: str): return f''' from eth2spec.deneb import {preset_name} as deneb -from eth2spec.utils.ssz.ssz_impl import serialize +from eth2spec.utils.ssz.ssz_impl import ssz_serialize, ssz_deserialize ''' @classmethod @@ -30,7 +30,7 @@ class NoopExecutionEngine(ExecutionEngine): def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload, parent_beacon_block_root: Root, - execution_requests_list: list[bytes]) -> bool: + execution_requests_list: Sequence[bytes]) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, diff --git a/setup.py b/setup.py index f976778dc9..83ae7accb4 100644 --- a/setup.py +++ b/setup.py @@ -561,7 +561,7 @@ def run(self): python_requires=">=3.9, <4", extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], - "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"], + "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==3.3.1"], "generator": ["setuptools>=72.0.0", "pytest>4.4", "python-snappy==0.7.3", "filelock", "pathos==0.3.0"], "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3", "mkdocs-awesome-pages-plugin==2.8.0"] }, @@ -572,7 +572,7 @@ def run(self): "py_ecc==6.0.0", "milagro_bls_binding==1.9.0", "remerkleable==0.1.28", - "trie==2.0.2", + "trie>=3,<4", RUAMEL_YAML_VERSION, "lru-dict==1.2.0", MARKO_VERSION, diff --git a/solidity_deposit_contract/Makefile b/solidity_deposit_contract/Makefile new file mode 100644 index 0000000000..a353d931bb --- /dev/null +++ b/solidity_deposit_contract/Makefile @@ -0,0 +1,42 @@ +SOLIDITY_FILE_NAME = deposit_contract.json +SOLIDITY_DEPOSIT_CONTRACT_SOURCE = deposit_contract.sol +DEPOSIT_CONTRACT_TESTER_DIR = web3_tester + +export DAPP_SKIP_BUILD:=1 +export DAPP_SRC:=$(CURDIR) +export DAPP_LIB:=$(CURDIR)/lib +export DAPP_JSON:=build/combined.json + +all: \ + compile_deposit_contract \ + install_deposit_contract_web3_tester \ + test_deposit_contract_web3_tests + +compile_deposit_contract: + @git submodule update --recursive --init + @solc --metadata-literal --optimize --optimize-runs 5000000 --bin --abi \ + --combined-json=abi,bin,bin-runtime,srcmap,srcmap-runtime,ast,metadata,storage-layout \ + --overwrite -o build $(SOLIDITY_DEPOSIT_CONTRACT_SOURCE) tests/deposit_contract.t.sol + @/bin/echo -n '{"abi": ' > $(SOLIDITY_FILE_NAME) + @cat build/DepositContract.abi >> $(SOLIDITY_FILE_NAME) + @/bin/echo -n ', "bytecode": "0x' >> $(SOLIDITY_FILE_NAME) + @cat build/DepositContract.bin >> $(SOLIDITY_FILE_NAME) + @/bin/echo -n '"}' >> $(SOLIDITY_FILE_NAME) + +test_deposit_contract: + @dapp test -v --fuzz-runs 5 + +install_deposit_contract_web3_tester: + @cd $(DEPOSIT_CONTRACT_TESTER_DIR); \ + python3 -m venv venv; \ + source venv/bin/activate; \ + python3 -m pip install -r ../../requirements_preinstallation.txt; \ + python3 -m pip install -r requirements.txt + +test_deposit_contract_web3_tests: + @cd $(DEPOSIT_CONTRACT_TESTER_DIR); \ + source venv/bin/activate; \ + python3 -m pytest . + +clean: + @git clean -fdx diff --git a/solidity_deposit_contract/README.md b/solidity_deposit_contract/README.md index 0388d7d2f5..298ea92fef 100644 --- a/solidity_deposit_contract/README.md +++ b/solidity_deposit_contract/README.md @@ -13,7 +13,7 @@ In August 2020, version `r2` was released with metadata modifications and relice ## Compiling solidity deposit contract -In the `eth2.0-specs` directory run: +In this directory run: ```sh make compile_deposit_contract ``` @@ -31,8 +31,8 @@ solc --optimize --optimize-runs 5000000 --metadata-literal --bin deposit_contrac ## Running web3 tests -1. In the `eth2.0-specs` directory run `make install_deposit_contract_web3_tester` to install the tools needed (make sure to have Python 3.7 and pip installed). -2. In the `eth2.0-specs` directory run `make test_deposit_contract_web3_tests` to execute the tests. +1. In this directory run `make install_deposit_contract_web3_tester` to install the tools needed (make sure to have Python 3 and pip installed). +2. In this directory run `make test_deposit_contract_web3_tests` to execute the tests. ## Running randomized `dapp` tests: diff --git a/solidity_deposit_contract/web3_tester/requirements.txt b/solidity_deposit_contract/web3_tester/requirements.txt index 8dadab0bc2..92c8298bfb 100644 --- a/solidity_deposit_contract/web3_tester/requirements.txt +++ b/solidity_deposit_contract/web3_tester/requirements.txt @@ -1,5 +1,5 @@ -eth-tester[py-evm]>=0.3.0b1,<0.4 -web3==5.4.0 -pytest==3.6.1 +eth-tester[py-evm]>=0.12.0b1 +web3>=6.11.0 +pytest>=4.4 # The eth2spec ../../ diff --git a/solidity_deposit_contract/web3_tester/tests/conftest.py b/solidity_deposit_contract/web3_tester/tests/conftest.py index 57b5f6b7a7..31686eabf3 100644 --- a/solidity_deposit_contract/web3_tester/tests/conftest.py +++ b/solidity_deposit_contract/web3_tester/tests/conftest.py @@ -51,7 +51,7 @@ def registration_contract(w3, tester): abi=contract_abi, bytecode=contract_bytecode) tx_hash = registration.constructor().transact() - tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash) + tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) registration_deployed = w3.eth.contract( address=tx_receipt.contractAddress, abi=contract_abi diff --git a/solidity_deposit_contract/web3_tester/tests/test_deposit.py b/solidity_deposit_contract/web3_tester/tests/test_deposit.py index 4b16446a16..e4b794ffbf 100644 --- a/solidity_deposit_contract/web3_tester/tests/test_deposit.py +++ b/solidity_deposit_contract/web3_tester/tests/test_deposit.py @@ -2,7 +2,7 @@ import pytest import eth_utils -from eth2spec.phase0.spec import DepositData +from eth2spec.phase0.mainnet import DepositData from eth2spec.utils.ssz.ssz_typing import List from eth2spec.utils.ssz.ssz_impl import hash_tree_root @@ -119,7 +119,7 @@ def test_deposit_inputs(registration_contract, def test_deposit_event_log(registration_contract, a0, w3): - log_filter = registration_contract.events.DepositEvent.createFilter( + log_filter = registration_contract.events.DepositEvent.create_filter( fromBlock='latest', ) deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] @@ -154,7 +154,7 @@ def test_deposit_event_log(registration_contract, a0, w3): def test_deposit_tree(registration_contract, w3, assert_tx_failed): - log_filter = registration_contract.events.DepositEvent.createFilter( + log_filter = registration_contract.events.DepositEvent.create_filter( fromBlock='latest', ) @@ -178,7 +178,7 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed): tx_hash = registration_contract.functions.deposit( *deposit_input, ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) - receipt = w3.eth.getTransactionReceipt(tx_hash) + receipt = w3.eth.get_transaction_receipt(tx_hash) print("deposit transaction consumes %d gas" % receipt['gasUsed']) logs = log_filter.get_new_entries() diff --git a/specs/_features/custody_game/validator.md b/specs/_features/custody_game/validator.md index 37df35e882..e711d92a1c 100644 --- a/specs/_features/custody_game/validator.md +++ b/specs/_features/custody_game/validator.md @@ -1,7 +1,7 @@ # Custody Game -- Honest Validator **Notice**: This document is a work-in-progress for researchers and implementers. -This is an accompanying document to the [Custody Game](./), which describes the expected actions of a "validator" +This is an accompanying document to [Custody Game -- The Beacon Chain](./beacon-chain.md), which describes the expected actions of a "validator" participating in the shard data Custody Game. ## Table of contents diff --git a/specs/_features/eip6800/fork.md b/specs/_features/eip6800/fork.md index 74f143f597..14172c9f3e 100644 --- a/specs/_features/eip6800/fork.md +++ b/specs/_features/eip6800/fork.md @@ -68,7 +68,7 @@ If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == an irregular state change is made to upgrade to eip6800. The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `EIP6800_FORK_EPOCH * SLOTS_PER_EPOCH`. -Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document. +Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document. In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead, the logic must be within `process_slots`. ```python diff --git a/specs/_features/eip6914/beacon-chain.md b/specs/_features/eip6914/beacon-chain.md index 1e0b20747e..fcb7716f7e 100644 --- a/specs/_features/eip6914/beacon-chain.md +++ b/specs/_features/eip6914/beacon-chain.md @@ -23,7 +23,7 @@ This is the beacon chain specification to assign new deposits to existing validator records. Refers to [EIP-6914](https://github.com/ethereum/EIPs/pull/6914). -*Note:* This specification is built upon [Capella](../../capella/beacon_chain.md) and is under active development. +*Note:* This specification is built upon [Capella](../../capella/beacon-chain.md) and is under active development. ## Preset diff --git a/specs/_features/eip7594/fork.md b/specs/_features/eip7594/fork.md index 790ab0287d..4f2c17561f 100644 --- a/specs/_features/eip7594/fork.md +++ b/specs/_features/eip7594/fork.md @@ -44,6 +44,8 @@ def compute_fork_version(epoch: Epoch) -> Version: """ if epoch >= EIP7594_FORK_EPOCH: return EIP7594_FORK_VERSION + if epoch >= ELECTRA_FORK_EPOCH: + return ELECTRA_FORK_VERSION if epoch >= DENEB_FORK_EPOCH: return DENEB_FORK_VERSION if epoch >= CAPELLA_FORK_EPOCH: @@ -71,8 +73,8 @@ If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == an irregular state change is made to upgrade to EIP7594. ```python -def upgrade_to_eip7594(pre: deneb.BeaconState) -> BeaconState: - epoch = deneb.get_current_epoch(pre) +def upgrade_to_eip7594(pre: electra.BeaconState) -> BeaconState: + epoch = electra.get_current_epoch(pre) post = BeaconState( # Versioning genesis_time=pre.genesis_time, @@ -119,6 +121,17 @@ def upgrade_to_eip7594(pre: deneb.BeaconState) -> BeaconState: next_withdrawal_validator_index=pre.next_withdrawal_validator_index, # Deep history valid from Capella onwards historical_summaries=pre.historical_summaries, + # On-chain deposits + deposit_requests_start_index=pre.deposit_requests_start_index, + # Consolidations + deposit_balance_to_consume=pre.deposit_balance_to_consume, + exit_balance_to_consume=pre.exit_balance_to_consume, + earliest_exit_epoch=pre.earliest_exit_epoch, + consolidation_balance_to_consume=pre.consolidation_balance_to_consume, + earliest_consolidation_epoch=pre.earliest_consolidation_epoch, + pending_balance_deposits=pre.pending_balance_deposits, + pending_partial_withdrawals=pre.pending_partial_withdrawals, + pending_consolidations=pre.pending_consolidations, ) return post diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index 4c4ae83efb..c989711807 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -223,7 +223,7 @@ Requests sidecars by block root and index. The response is a list of `DataColumnIdentifier` whose length is less than or equal to the number of requests. It may be less in the case that the responding peer is missing blocks or sidecars. -Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. +Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted through `verify_data_column_sidecar`, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. No more than `MAX_REQUEST_DATA_COLUMN_SIDECARS` may be requested at a time. @@ -269,7 +269,7 @@ Response Content: Requests data column sidecars in the slot range `[start_slot, start_slot + count)` of the given `columns`, leading up to the current head block as selected by fork choice. -Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted, has valid inclusion proof, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. +Before consuming the next response chunk, the response reader SHOULD verify the data column sidecar is well-formatted through `verify_data_column_sidecar`, has valid inclusion proof through `verify_data_column_sidecar_inclusion_proof`, and is correct w.r.t. the expected KZG commitments through `verify_data_column_sidecar_kzg_proofs`. `DataColumnSidecarsByRange` is primarily used to sync data columns that may have been missed on gossip and to sync within the `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` window. diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 41e2fa9d3f..aa63ebe87e 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -34,6 +34,7 @@ The specification of these changes continues in the same format as the network s - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) - [BlobSidecarsByRoot v1](#blobsidecarsbyroot-v1) + - [Blob retrieval via local execution layer client](#blob-retrieval-via-local-execution-layer-client) - [BlobSidecarsByRange v1](#blobsidecarsbyrange-v1) - [Design decision rationale](#design-decision-rationale) - [Why are blobs relayed as a sidecar, separate from beacon blocks?](#why-are-blobs-relayed-as-a-sidecar-separate-from-beacon-blocks) @@ -309,6 +310,15 @@ Clients SHOULD include a sidecar in the response as soon as it passes the gossip Clients SHOULD NOT respond with sidecars related to blocks that fail gossip validation rules. Clients SHOULD NOT respond with sidecars related to blocks that fail the beacon chain state transition +###### Blob retrieval via local execution layer client + +In addition to `BlobSidecarsByRoot` requests, recent blobs MAY be retrieved by querying the Execution Layer (i.e. via `engine_getBlobsV1`). +Implementers are encouraged to leverage this method to increase the likelihood of incorporating and attesting to the last block when its proposer is not able to publish blobs on time. + +When clients use the local execution layer to retrieve blobs, they MUST behave as if the corresponding `blob_sidecar` had been received via gossip. In particular they MUST: +* publish the corresponding `blob_sidecar` on the `blob_sidecar_{subnet_id}` subnet. +* update gossip rule related data structures (i.e. update the anti-equivocation cache). + ##### BlobSidecarsByRange v1 **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` diff --git a/specs/electra/beacon-chain.md b/specs/electra/beacon-chain.md index 7c6d9fe1fa..3e12ab2171 100644 --- a/specs/electra/beacon-chain.md +++ b/specs/electra/beacon-chain.md @@ -24,12 +24,13 @@ - [Validator cycle](#validator-cycle) - [Containers](#containers) - [New containers](#new-containers) - - [`DepositRequest`](#depositrequest) - [`PendingDeposit`](#pendingdeposit) - [`PendingPartialWithdrawal`](#pendingpartialwithdrawal) + - [`PendingConsolidation`](#pendingconsolidation) + - [`DepositRequest`](#depositrequest) - [`WithdrawalRequest`](#withdrawalrequest) - [`ConsolidationRequest`](#consolidationrequest) - - [`PendingConsolidation`](#pendingconsolidation) + - [`SingleAttestation`](#singleattestation) - [`ExecutionRequests`](#executionrequests) - [Modified Containers](#modified-containers) - [`AttesterSlashing`](#attesterslashing) @@ -118,7 +119,7 @@ Electra is a consensus-layer upgrade containing a number of features. Including: * [EIP-7251](https://eips.ethereum.org/EIPS/eip-7251): Increase the MAX_EFFECTIVE_BALANCE * [EIP-7549](https://eips.ethereum.org/EIPS/eip-7549): Move committee index outside Attestation -*Note:* This specification is built upon [Deneb](../deneb/beacon_chain.md) and is under active development. +*Note:* This specification is built upon [Deneb](../deneb/beacon-chain.md) and is under active development. ## Constants @@ -201,19 +202,6 @@ The following values are (non-configurable) constants used throughout the specif ### New containers -#### `DepositRequest` - -*Note*: The container is new in EIP6110. - -```python -class DepositRequest(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 - amount: Gwei - signature: BLSSignature - index: uint64 -``` - #### `PendingDeposit` *Note*: The container is new in EIP7251. @@ -237,6 +225,30 @@ class PendingPartialWithdrawal(Container): amount: Gwei withdrawable_epoch: Epoch ``` + +#### `PendingConsolidation` + +*Note*: The container is new in EIP7251. + +```python +class PendingConsolidation(Container): + source_index: ValidatorIndex + target_index: ValidatorIndex +``` + +#### `DepositRequest` + +*Note*: The container is new in EIP6110. + +```python +class DepositRequest(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 + amount: Gwei + signature: BLSSignature + index: uint64 +``` + #### `WithdrawalRequest` *Note*: The container is new in EIP7251:EIP7002. @@ -259,14 +271,14 @@ class ConsolidationRequest(Container): target_pubkey: BLSPubkey ``` -#### `PendingConsolidation` - -*Note*: The container is new in EIP7251. +#### `SingleAttestation` ```python -class PendingConsolidation(Container): - source_index: ValidatorIndex - target_index: ValidatorIndex +class SingleAttestation(Container): + committee_index: CommitteeIndex + attester_index: ValidatorIndex + data: AttestationData + signature: BLSSignature ``` #### `ExecutionRequests` @@ -572,10 +584,12 @@ def get_attesting_indices(state: BeaconState, attestation: Attestation) -> Set[V output: Set[ValidatorIndex] = set() committee_indices = get_committee_indices(attestation.committee_bits) committee_offset = 0 - for index in committee_indices: - committee = get_beacon_committee(state, attestation.data.slot, index) + for committee_index in committee_indices: + committee = get_beacon_committee(state, attestation.data.slot, committee_index) committee_attesters = set( - index for i, index in enumerate(committee) if attestation.aggregation_bits[committee_offset + i]) + attester_index for i, attester_index in enumerate(committee) + if attestation.aggregation_bits[committee_offset + i] + ) output = output.union(committee_attesters) committee_offset += len(committee) @@ -992,9 +1006,9 @@ class NewPayloadRequest(object): def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload, parent_beacon_block_root: Root, - execution_requests_list: list[bytes]) -> bool: + execution_requests_list: Sequence[bytes]) -> bool: """ - Return ``True`` if and only if ``execution_payload`` and ``execution_requests`` + Return ``True`` if and only if ``execution_payload`` and ``execution_requests`` are valid with respect to ``self.execution_state``. """ ... @@ -1056,7 +1070,7 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], withdrawal_index = state.next_withdrawal_index validator_index = state.next_withdrawal_validator_index withdrawals: List[Withdrawal] = [] - partial_withdrawals_count = 0 + processed_partial_withdrawals_count = 0 # [New in Electra:EIP7251] Consume pending partial withdrawals for withdrawal in state.pending_partial_withdrawals: @@ -1076,13 +1090,16 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], )) withdrawal_index += WithdrawalIndex(1) - partial_withdrawals_count += 1 + processed_partial_withdrawals_count += 1 # Sweep for remaining. bound = min(len(state.validators), MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) for _ in range(bound): validator = state.validators[validator_index] - balance = state.balances[validator_index] + # [Modified in Electra:EIP7251] + partially_withdrawn_balance = sum( + withdrawal.amount for withdrawal in withdrawals if withdrawal.validator_index == validator_index) + balance = state.balances[validator_index] - partially_withdrawn_balance if is_fully_withdrawable_validator(validator, balance, epoch): withdrawals.append(Withdrawal( index=withdrawal_index, @@ -1102,7 +1119,7 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], if len(withdrawals) == MAX_WITHDRAWALS_PER_PAYLOAD: break validator_index = ValidatorIndex((validator_index + 1) % len(state.validators)) - return withdrawals, partial_withdrawals_count + return withdrawals, processed_partial_withdrawals_count ``` ##### Modified `process_withdrawals` @@ -1111,7 +1128,8 @@ def get_expected_withdrawals(state: BeaconState) -> Tuple[Sequence[Withdrawal], ```python def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: - expected_withdrawals, partial_withdrawals_count = get_expected_withdrawals(state) # [Modified in Electra:EIP7251] + # [Modified in Electra:EIP7251] + expected_withdrawals, processed_partial_withdrawals_count = get_expected_withdrawals(state) assert payload.withdrawals == expected_withdrawals @@ -1119,7 +1137,7 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: decrease_balance(state, withdrawal.validator_index, withdrawal.amount) # Update pending partial withdrawals [New in Electra:EIP7251] - state.pending_partial_withdrawals = state.pending_partial_withdrawals[partial_withdrawals_count:] + state.pending_partial_withdrawals = state.pending_partial_withdrawals[processed_partial_withdrawals_count:] # Update the next withdrawal index if this block contained withdrawals if len(expected_withdrawals) != 0: @@ -1145,10 +1163,10 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: *Note*: Encodes execution requests as defined by [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). ```python -def get_execution_requests_list(execution_requests: ExecutionRequests) -> list[bytes]: - deposit_bytes = serialize(execution_requests.deposits) - withdrawal_bytes = serialize(execution_requests.withdrawals) - consolidation_bytes = serialize(execution_requests.consolidations) +def get_execution_requests_list(execution_requests: ExecutionRequests) -> Sequence[bytes]: + deposit_bytes = ssz_serialize(execution_requests.deposits) + withdrawal_bytes = ssz_serialize(execution_requests.withdrawals) + consolidation_bytes = ssz_serialize(execution_requests.consolidations) return [deposit_bytes, withdrawal_bytes, consolidation_bytes] ``` @@ -1290,11 +1308,12 @@ def get_validator_from_deposit(pubkey: BLSPubkey, withdrawal_credentials: Bytes3 validator = Validator( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, + effective_balance=Gwei(0), + slashed=False, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=Gwei(0), ) # [Modified in Electra:EIP7251] @@ -1618,6 +1637,12 @@ def process_consolidation_request( return if target_validator.exit_epoch != FAR_FUTURE_EPOCH: return + # Verify the source has been active long enough + if current_epoch < source_validator.activation_epoch + SHARD_COMMITTEE_PERIOD: + return + # Verify the source has no pending withdrawals in the queue + if get_pending_balance_to_withdraw(state, source_index) > 0: + return # Initiate source validator exit and append pending consolidation source_validator.exit_epoch = compute_consolidation_epoch_and_update_churn( diff --git a/specs/electra/light-client/full-node.md b/specs/electra/light-client/full-node.md deleted file mode 100644 index 0393aaec24..0000000000 --- a/specs/electra/light-client/full-node.md +++ /dev/null @@ -1,74 +0,0 @@ -# Electra Light Client -- Full Node - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Introduction](#introduction) -- [Helper functions](#helper-functions) - - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header) - - - - -## Introduction - -Execution payload data is updated to account for the Electra upgrade. - -## Helper functions - -### Modified `block_to_light_client_header` - -```python -def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: - epoch = compute_epoch_at_slot(block.message.slot) - - if epoch >= CAPELLA_FORK_EPOCH: - payload = block.message.body.execution_payload - execution_header = ExecutionPayloadHeader( - parent_hash=payload.parent_hash, - fee_recipient=payload.fee_recipient, - state_root=payload.state_root, - receipts_root=payload.receipts_root, - logs_bloom=payload.logs_bloom, - prev_randao=payload.prev_randao, - block_number=payload.block_number, - gas_limit=payload.gas_limit, - gas_used=payload.gas_used, - timestamp=payload.timestamp, - extra_data=payload.extra_data, - base_fee_per_gas=payload.base_fee_per_gas, - block_hash=payload.block_hash, - transactions_root=hash_tree_root(payload.transactions), - withdrawals_root=hash_tree_root(payload.withdrawals), - ) - if epoch >= DENEB_FORK_EPOCH: - execution_header.blob_gas_used = payload.blob_gas_used - execution_header.excess_blob_gas = payload.excess_blob_gas - - execution_branch = ExecutionBranch( - compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_GINDEX)) - else: - # Note that during fork transitions, `finalized_header` may still point to earlier forks. - # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), - # it was not included in the corresponding light client data. To ensure compatibility - # with legacy data going through `upgrade_lc_header_to_capella`, leave out execution data. - execution_header = ExecutionPayloadHeader() - execution_branch = ExecutionBranch() - - return LightClientHeader( - beacon=BeaconBlockHeader( - slot=block.message.slot, - proposer_index=block.message.proposer_index, - parent_root=block.message.parent_root, - state_root=block.message.state_root, - body_root=hash_tree_root(block.message.body), - ), - execution=execution_header, - execution_branch=execution_branch, - ) -``` diff --git a/specs/electra/light-client/sync-protocol.md b/specs/electra/light-client/sync-protocol.md index 26da70ee34..f042119b00 100644 --- a/specs/electra/light-client/sync-protocol.md +++ b/specs/electra/light-client/sync-protocol.md @@ -18,7 +18,6 @@ - [Modified `current_sync_committee_gindex_at_slot`](#modified-current_sync_committee_gindex_at_slot) - [Modified `next_sync_committee_gindex_at_slot`](#modified-next_sync_committee_gindex_at_slot) - [Modified `get_lc_execution_root`](#modified-get_lc_execution_root) - - [Modified `is_valid_light_client_header`](#modified-is_valid_light_client_header) @@ -152,28 +151,3 @@ def get_lc_execution_root(header: LightClientHeader) -> Root: return Root() ``` - -### Modified `is_valid_light_client_header` - -```python -def is_valid_light_client_header(header: LightClientHeader) -> bool: - epoch = compute_epoch_at_slot(header.beacon.slot) - - if epoch < DENEB_FORK_EPOCH: - if header.execution.blob_gas_used != uint64(0) or header.execution.excess_blob_gas != uint64(0): - return False - - if epoch < CAPELLA_FORK_EPOCH: - return ( - header.execution == ExecutionPayloadHeader() - and header.execution_branch == ExecutionBranch() - ) - - return is_valid_merkle_branch( - leaf=get_lc_execution_root(header), - branch=header.execution_branch, - depth=floorlog2(EXECUTION_PAYLOAD_GINDEX), - index=get_subtree_index(EXECUTION_PAYLOAD_GINDEX), - root=header.beacon.body_root, - ) -``` diff --git a/specs/electra/p2p-interface.md b/specs/electra/p2p-interface.md index fca877a8aa..0ea33df9f7 100644 --- a/specs/electra/p2p-interface.md +++ b/specs/electra/p2p-interface.md @@ -56,9 +56,18 @@ The following validations are added: ##### `beacon_attestation_{subnet_id}` -The following convenience variables are re-defined -- `index = get_committee_indices(attestation.committee_bits)[0]` +The topic is updated to propagate `SingleAttestation` objects. + +The following convenience variables are re-defined: +- `index = attestation.committee_index` The following validations are added: -* [REJECT] `len(committee_indices) == 1`, where `committee_indices = get_committee_indices(attestation)`. -* [REJECT] `attestation.data.index == 0` +- _[REJECT]_ `attestation.data.index == 0` +- _[REJECT]_ The attester is a member of the committee -- i.e. + `attestation.attester_index in get_beacon_committee(state, attestation.data.slot, index)`. + +The following validations are removed: +- _[REJECT]_ The attestation is unaggregated -- + that is, it has exactly one participating validator (`len([bit for bit in aggregation_bits if bit]) == 1`, i.e. exactly 1 bit is set). +- _[REJECT]_ The number of aggregation bits matches the committee size -- i.e. + `len(aggregation_bits) == len(get_beacon_committee(state, attestation.data.slot, index))`. diff --git a/specs/electra/validator.md b/specs/electra/validator.md index f589e963c5..0e26dedf94 100644 --- a/specs/electra/validator.md +++ b/specs/electra/validator.md @@ -8,16 +8,22 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Helpers](#helpers) + - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) - [Containers](#containers) - [Modified Containers](#modified-containers) - [`AggregateAndProof`](#aggregateandproof) - [`SignedAggregateAndProof`](#signedaggregateandproof) +- [Protocol](#protocol) + - [`ExecutionEngine`](#executionengine) + - [Modified `get_payload`](#modified-get_payload) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - [Attester slashings](#attester-slashings) - [Attestations](#attestations) - [Deposits](#deposits) - [Execution payload](#execution-payload) + - [Execution Requests](#execution-requests) - [Attesting](#attesting) - [Construct attestation](#construct-attestation) - [Attestation aggregation](#attestation-aggregation) @@ -32,12 +38,25 @@ This document represents the changes to be made in the code of an "honest valida ## Prerequisites -This document is an extension of the [Deneb -- Honest Validator](../../deneb/validator.md) guide. +This document is an extension of the [Deneb -- Honest Validator](../deneb/validator.md) guide. All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [Electra](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. +## Helpers + +### Modified `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload + block_value: uint256 + blobs_bundle: BlobsBundle + execution_requests: Sequence[bytes] # [New in Electra] +``` + ## Containers ### Modified Containers @@ -59,6 +78,24 @@ class SignedAggregateAndProof(Container): signature: BLSSignature ``` +## Protocol + +### `ExecutionEngine` + +#### Modified `get_payload` + +Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that +has been built since the corresponding call to `notify_forkchoice_updated` method. + +```python +def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + """ + Return ExecutionPayload, uint256, BlobsBundle and execution requests (as Sequence[bytes]) objects. + """ + # pylint: disable=unused-argument + ... +``` + ## Block proposal ### Constructing the `BeaconBlockBody` @@ -148,15 +185,34 @@ def prepare_execution_payload(state: BeaconState, ) ``` +#### Execution Requests + +*[New in Electra]* + +1. The execution payload is obtained from the execution engine as defined above using `payload_id`. The response also includes a `execution_requests` entry containing a list of bytes. Each element on the list corresponds to one SSZ list of requests as defined +in [EIP-7685](https://eips.ethereum.org/EIPS/eip-7685). The index of each element in the array determines the type of request. +2. Set `block.body.execution_requests = get_execution_requests(execution_requests)`, where: + +```python +def get_execution_requests(execution_requests: Sequence[bytes]) -> ExecutionRequests: + deposits = ssz_deserialize(List[DepositRequest, MAX_DEPOSIT_REQUESTS_PER_PAYLOAD], execution_requests[0]) + withdrawals = ssz_deserialize(List[WithdrawalRequest, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD], execution_requests[1]) + consolidations = ssz_deserialize(List[ConsolidationRequest, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD], + execution_requests[2]) + + return ExecutionRequests(deposits, withdrawals, consolidations) +``` + ## Attesting ### Construct attestation -- Set `attestation_data.index = 0`. -- Let `attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]` of length `len(committee)`, where the bit of the index of the validator in the `committee` is set to `0b1`. -- Let `attestation.committee_bits` be a `Bitvector[MAX_COMMITTEES_PER_SLOT]`, where the bit at the index associated with the validator's committee is set to `0b1`. +The validator creates `attestation` as a `SingleAttestation` container +with updated field assignments: -*Note*: Calling `get_attesting_indices(state, attestation)` should return a list of length equal to 1, containing `validator_index`. +- Set `attestation_data.index = 0`. +- Set `attestation.committee_index` to the index associated with the validator's committee. +- Set `attestation.attester_index` to the index of the validator. ## Attestation aggregation @@ -164,4 +220,4 @@ def prepare_execution_payload(state: BeaconState, - Set `attestation_data.index = 0`. - Let `aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE * MAX_COMMITTEES_PER_SLOT]` of length `len(committee)`, where each bit set from each individual attestation is set to `0b1`. -- Set `attestation.committee_bits = committee_bits`, where `committee_bits` has the same value as in each individual attestation. +- Set `attestation.committee_bits = committee_bits`, where `committee_bits` has the bit set corresponding to `committee_index` in each individual attestation. diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 9d5f8446a0..3d860d4a3e 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -1854,11 +1854,12 @@ def get_validator_from_deposit(pubkey: BLSPubkey, withdrawal_credentials: Bytes3 return Validator( pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, + effective_balance=effective_balance, + slashed=False, activation_eligibility_epoch=FAR_FUTURE_EPOCH, activation_epoch=FAR_FUTURE_EPOCH, exit_epoch=FAR_FUTURE_EPOCH, withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=effective_balance, ) ``` diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bc29b9ad92..0e1953946e 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -94,6 +94,7 @@ It consists of four main sections: - [Why are messages length-prefixed with a protobuf varint in the SSZ-encoding?](#why-are-messages-length-prefixed-with-a-protobuf-varint-in-the-ssz-encoding) - [Why do we version protocol strings with ordinals instead of semver?](#why-do-we-version-protocol-strings-with-ordinals-instead-of-semver) - [Why is it called Req/Resp and not RPC?](#why-is-it-called-reqresp-and-not-rpc) + - [What is a typical rate limiting strategy?](#what-is-a-typical-rate-limiting-strategy) - [Why do we allow empty responses in block requests?](#why-do-we-allow-empty-responses-in-block-requests) - [Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from?](#why-does-beaconblocksbyrange-let-the-server-choose-which-branch-to-send-blocks-from) - [Why are `BlocksByRange` requests only required to be served for the latest `MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs?](#why-are-blocksbyrange-requests-only-required-to-be-served-for-the-latest-min_epochs_for_block_requests-epochs) @@ -194,8 +195,6 @@ This section outlines configurations that are used in this spec. | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | -| `RESP_TIMEOUT` | `10` | The maximum duration in **seconds** for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | | `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500` | The maximum **milliseconds** of clock disparity assumed between honest nodes. | | `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | @@ -204,6 +203,7 @@ This section outlines configurations that are used in this spec. | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `MAX_CONCURRENT_REQUESTS` | `2` | Maximum number of concurrent requests per protocol ID that a client may issue | ### MetaData @@ -561,10 +561,9 @@ The request MUST be encoded according to the encoding strategy. The requester MUST close the write side of the stream once it finishes writing the request message. At this point, the stream will be half-closed. -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). -On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. +The requester MUST NOT make more than `MAX_CONCURRENT_REQUESTS` concurrent requests with the same protocol ID. -If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. +If a timeout occurs or the response is no longer relevant, the requester SHOULD reset the stream. A requester SHOULD read from the stream until either: 1. An error result is received in one of the chunks (the error payload MAY be read before stopping). @@ -593,10 +592,10 @@ The responder MUST: If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. -The entire request should be read in no more than `RESP_TIMEOUT`. -Upon a timeout, the responder SHOULD reset the stream. +The responder MAY rate-limit chunks by withholding each chunk until capacity is available. The responder MUST NOT respond with an error or close the stream when rate limiting. + +When rate limiting, the responder MUST send each `response_chunk` in full promptly but may introduce delays between each chunk. -The responder SHOULD send a `response_chunk` promptly. Chunks start with a **single-byte** response code which determines the contents of the `response_chunk` (`result` particle in the BNF grammar above). For multiple chunks, only the last chunk is allowed to have a non-zero error code (i.e. The chunk stream is terminated once an error occurs). @@ -626,6 +625,8 @@ The `ErrorMessage` schema is: *Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. +The responder MAY penalise peers that concurrently open more than `MAX_CONCURRENT_REQUESTS` streams for the same request type, for the protocol IDs defined in this specification. + #### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. @@ -1455,6 +1456,20 @@ For this reason, we remove and replace semver with ordinals that require explici Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. +#### What is a typical rate limiting strategy? + +The responder typically will want to rate limit requests to protect against spam and to manage resource consumption, while the requester will want to maximise performance based on its own resource allocation strategy. For the network, it is beneficial if available resources are used optimally. + +Broadly, the requester does not know the capacity / limit of each server but can derive it from the rate of responses for the purpose of selecting the next peer for a request. + +Because the server withholds the response until capacity is available, a client can optimistically send requests without risking running into negative scoring situations or sub-optimal rate polling. + +A typical approach for the requester is to implement a timeout on the request that depends on the nature of the request and on connectivity parameters in general - for example when requesting blocks, a peer might choose to send a request to a second peer if the first peer does not respond within a reasonable time, and to reset the request to the first peer if the second peer responds faster. Clients may use past response performance to reward fast peers when implementing peer scoring. + +A typical approach for the responder is to implement a two-level token/leaky bucket with a per-peer limit and a global limit. The granularity of rate limiting may be based either on full requests or individual chunks with the latter being preferable. A token cost may be assigned to the request itself and separately each chunk in the response so as to remain protected both against large and frequent requests. + +For requesters, rate limiting is not distinguishable from other conditions causing slow responses (slow peers, congestion etc) and since the latter conditions must be handled anyway, including rate limiting in this strategy keeps the implementation simple. + #### Why do we allow empty responses in block requests? When requesting blocks by range or root, it may happen that there are no blocks in the selected range or the responding node does not have the requested blocks. @@ -1486,10 +1501,10 @@ If a request for the parent fails, it's indicative of poor peer quality since pe When connecting, the `Status` message gives an idea about the sync status of a particular peer, but this changes over time. By the time a subsequent `BeaconBlockByRange` request is processed, the information may be stale, -and the responding side might have moved on to a new finalization point and pruned blocks around the previous head and finalized blocks. +and the responder might have moved on to a new finalization point and pruned blocks around the previous head and finalized blocks. -To avoid this race condition, we allow the responding side to choose which branch to send to the requesting client. -The requesting client then goes on to validate the blocks and incorporate them in their own database +To avoid this race condition, we allow the responder to choose which branch to send to the requester. +The requester then goes on to validate the blocks and incorporate them in their own database -- because they follow the same rules, they should at this point arrive at the same canonical chain. #### Why are `BlocksByRange` requests only required to be served for the latest `MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs? diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 93244d44a1..e3af99e3c0 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.5.0-alpha.8 +1.5.0-alpha.9 diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 85ccec017a..f8836c8730 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -65,7 +65,14 @@ def get_default_yaml(): def _represent_none(self, _): return self.represent_scalar('tag:yaml.org,2002:null', 'null') + def _represent_str(self, data): + if data.startswith("0x"): + # Without this, a zero-byte hex string is represented without quotes. + return self.represent_scalar('tag:yaml.org,2002:str', data, style="'") + return self.represent_str(data) + yaml.representer.add_representer(type(None), _represent_none) + yaml.representer.add_representer(str, _represent_str) return yaml @@ -187,14 +194,17 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): help="specify forks to run with. Allows all if no fork names are specified.", ) parser.add_argument( - "-c", - "--collect-only", + "--modcheck", action="store_true", default=False, - help="if set only print tests to generate, do not actually run the test and dump the target data", + help="check generator modules, do not run any tests.", ) - args = parser.parse_args() + + # Bail here if we are checking modules. + if args.modcheck: + return + output_dir = args.output_dir if not args.force: file_mode = "x" @@ -222,8 +232,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if len(presets) != 0: print(f"Filtering test-generator runs to only include forks: {', '.join(forks)}") - collect_only = args.collect_only - diagnostics_obj = Diagnostics() provider_start = time.time() @@ -231,9 +239,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): all_test_case_params = [] for tprov in test_providers: - if not collect_only: - # runs anything that we don't want to repeat for every test case. - tprov.prepare() + # Runs anything that we don't want to repeat for every test case. + tprov.prepare() for test_case in tprov.make_cases(): # If preset list is assigned, filter by presets. @@ -269,14 +276,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): provider_end = time.time() span = round(provider_end - provider_start, 2) - if collect_only: - print(f"Collected {diagnostics_obj.collected_test_count} tests in total") - else: - summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" - summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" - if span > TIME_THRESHOLD_TO_PRINT: - summary_message += f" in {span} seconds" - print(summary_message) + summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" + summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" + if span > TIME_THRESHOLD_TO_PRINT: + summary_message += f" in {span} seconds" + print(summary_message) diagnostics_output = { "collected_test_count": diagnostics_obj.collected_test_count, diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py index 153d6eee49..6f73f7155d 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py @@ -1,5 +1,6 @@ from importlib import import_module from inspect import getmembers, isfunction +from pkgutil import walk_packages from typing import Any, Callable, Dict, Iterable, Optional, List, Union from eth2spec.utils import bls @@ -134,3 +135,69 @@ def combine_mods(dict_1, dict_2): dict_3[key].append(dict_1[key]) return dict_3 + + +def check_mods(all_mods, pkg): + """ + Raise an exception if there is a missing/unexpected module in all_mods. + """ + def get_expected_modules(package, absolute=False): + """ + Return all modules (which are not packages) inside the given package. + """ + modules = [] + eth2spec = import_module('eth2spec') + prefix = eth2spec.__name__ + '.' + for _, modname, ispkg in walk_packages(eth2spec.__path__, prefix): + s = package if absolute else f'.{package}.' + if s in modname and not ispkg: + modules.append(modname) + return modules + + mods = [] + for fork in all_mods: + for mod in all_mods[fork].values(): + # If this key has a single value, normalize to list. + if isinstance(mod, str): + mod = [mod] + + # For each submodule, check if it is package. + # This is a "trick" we do to reuse a test format. + for sub in mod: + is_package = '.test_' not in sub + if is_package: + mods.extend(get_expected_modules(sub, absolute=True)) + else: + mods.append(sub) + + problems = [] + expected_mods = get_expected_modules(pkg) + if mods != expected_mods: + for e in expected_mods: + # Skip forks which are not in all_mods. + # The fork name is the 3rd item in the path. + fork = e.split('.')[2] + if fork not in all_mods: + continue + # Skip modules in the unittests package. + # These are not associated with generators. + if '.unittests.' in e: + continue + # The expected module is not in our list of modules. + # Add it to our list of problems. + if e not in mods: + problems.append('missing: ' + e) + + for t in mods: + # Skip helper modules. + # These do not define test functions. + if t.startswith('eth2spec.test.helpers'): + continue + # There is a module not defined in eth2spec. + # Add it to our list of problems. + if t not in expected_mods: + print('unexpected:', t) + problems.append('unexpected: ' + t) + + if problems: + raise Exception('[ERROR] module problems:\n ' + '\n '.join(problems)) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index 85175a3b9c..d0c4a6f22a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -10,8 +10,8 @@ expect_assertion_error, with_deneb_and_later ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -74,7 +74,7 @@ def test_incorrect_blob_tx_type(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type execution_payload.transactions = [opaque_tx] @@ -91,7 +91,7 @@ def test_incorrect_transaction_length_1_extra_byte(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx + b'\x12' # incorrect tx length, longer execution_payload.transactions = [opaque_tx] @@ -108,7 +108,7 @@ def test_incorrect_transaction_length_1_byte_short(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx[:-1] # incorrect tx length, shorter execution_payload.transactions = [opaque_tx] @@ -125,7 +125,7 @@ def test_incorrect_transaction_length_empty(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx[0:0] # incorrect tx length, empty execution_payload.transactions = [opaque_tx] @@ -142,7 +142,7 @@ def test_incorrect_transaction_length_32_extra_bytes(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length execution_payload.transactions = [opaque_tx] @@ -159,7 +159,7 @@ def test_no_transactions_with_commitments(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - _, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + _, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) @@ -175,7 +175,7 @@ def test_incorrect_commitment(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment execution_payload.transactions = [opaque_tx] @@ -192,7 +192,7 @@ def test_incorrect_commitments_order(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=Random(1111)) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=2, rng=Random(1111)) blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order execution_payload.transactions = [opaque_tx] @@ -206,7 +206,7 @@ def test_incorrect_commitments_order(spec, state): def test_incorrect_block_hash(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = b'\x12' * 32 # incorrect block hash @@ -223,7 +223,7 @@ def test_zeroed_commitment(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=1, is_valid_blob=False) assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) execution_payload.transactions = [opaque_tx] @@ -240,7 +240,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): """ execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) @@ -254,7 +254,7 @@ def test_invalid_correct_input__execution_invalid(spec, state): def test_invalid_exceed_max_blobs_per_block(spec, state): execution_payload = build_empty_execution_payload(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=spec.config.MAX_BLOBS_PER_BLOCK + 1) execution_payload.transactions = [opaque_tx] execution_payload.block_hash = compute_el_block_hash(spec, execution_payload, state) diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index a7e7f784e4..905cb390eb 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -25,14 +25,14 @@ from eth2spec.test.helpers.state import ( state_transition_and_sign_block, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) def get_block_with_blob(spec, state, rng=None): block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs = get_sample_opaque_tx(spec, blob_count=1, rng=rng) + opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs = get_sample_blob_tx(spec, blob_count=1, rng=rng) block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) block.body.blob_kzg_commitments = blob_kzg_commitments diff --git a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py index 1dda310123..6deccd31e2 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/deneb/merkle_proof/test_single_merkle_proof.py @@ -12,8 +12,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.debug.random_value import ( RandomizationMode, @@ -22,7 +22,7 @@ def _run_blob_kzg_commitment_merkle_proof_test(spec, state, rng=None): - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=1) + opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_blob_tx(spec, blob_count=1) if rng is None: block = build_empty_block_for_next_slot(spec, state) else: diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 94db2c34fc..b019175369 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -14,8 +14,8 @@ compute_el_block_hash, get_random_tx, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -27,7 +27,7 @@ def run_block_with_blobs(spec, state, blob_count, tx_count=1, blob_gas_used=1, e txs = [] blob_kzg_commitments = [] for _ in range(tx_count): - opaque_tx, _, commits, _ = get_sample_opaque_tx(spec, blob_count=blob_count) + opaque_tx, _, commits, _ = get_sample_blob_tx(spec, blob_count=blob_count) txs.append(opaque_tx) blob_kzg_commitments += commits diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py index fbff0f465e..ede2153824 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -7,7 +7,7 @@ expect_assertion_error, always_bls ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, get_poly_in_both_forms, eval_poly_in_coeff_form, diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 6724e8304a..0e02ab8c3c 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -6,8 +6,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -20,8 +20,8 @@ def _get_sample_sidecars(spec, state, rng): # 2 txs, each has 2 blobs blob_count = 2 - opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) - opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_opaque_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_1, blobs_1, blob_kzg_commitments_1, proofs_1 = get_sample_blob_tx(spec, blob_count=blob_count, rng=rng) + opaque_tx_2, blobs_2, blob_kzg_commitments_2, proofs_2 = get_sample_blob_tx(spec, blob_count=blob_count, rng=rng) assert opaque_tx_1 != opaque_tx_2 block.body.blob_kzg_commitments = blob_kzg_commitments_1 + blob_kzg_commitments_2 diff --git a/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py b/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py index 98c751508d..4bd5fb4912 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/merkle_proof/test_single_merkle_proof.py @@ -12,8 +12,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.debug.random_value import ( RandomizationMode, @@ -22,7 +22,7 @@ def _run_blob_kzg_commitments_merkle_proof_test(spec, state, rng=None): - opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1) + opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=1) if rng is None: block = build_empty_block_for_next_slot(spec, state) else: diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py index 625136b73e..dd0a80fda4 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py @@ -6,7 +6,7 @@ with_config_overrides, with_eip7594_and_later, ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, ) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py index 9a057d1018..1d72b142af 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -5,7 +5,7 @@ expect_assertion_error, with_eip7594_and_later, ) -from eth2spec.test.helpers.sharding import ( +from eth2spec.test.helpers.blob import ( get_sample_blob, ) from eth2spec.utils.bls import BLS_MODULUS diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py index 633508f9a2..931cc9c1d1 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py @@ -15,8 +15,8 @@ from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) @@ -25,7 +25,7 @@ def compute_data_column_sidecar(spec, state): rng = random.Random(5566) - opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2) + opaque_tx, blobs, blob_kzg_commitments, _ = get_sample_blob_tx(spec, blob_count=2) block = get_random_ssz_object( rng, spec.BeaconBlock, diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py index 49744946f0..8fdbb8e2e5 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_consolidation_request.py @@ -28,6 +28,8 @@ @spec_test @single_phase def test_basic_consolidation_in_current_consolidation_epoch(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -75,6 +77,8 @@ def test_basic_consolidation_in_current_consolidation_epoch(spec, state): @spec_test @single_phase def test_basic_consolidation_with_excess_target_balance(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -125,6 +129,8 @@ def test_basic_consolidation_with_excess_target_balance(spec, state): @spec_test @single_phase def test_basic_consolidation_with_excess_target_balance_and_compounding_credentials(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -175,6 +181,8 @@ def test_basic_consolidation_with_excess_target_balance_and_compounding_credenti @spec_test @single_phase def test_basic_consolidation_in_new_consolidation_epoch(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn # Set consolidation balance to consume to some arbitrary nonzero value below the churn limit state.consolidation_balance_to_consume = spec.EFFECTIVE_BALANCE_INCREMENT @@ -220,6 +228,8 @@ def test_basic_consolidation_in_new_consolidation_epoch(spec, state): @spec_test @single_phase def test_basic_consolidation_with_preexisting_churn(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -267,6 +277,8 @@ def test_basic_consolidation_with_preexisting_churn(spec, state): @spec_test @single_phase def test_basic_consolidation_with_insufficient_preexisting_churn(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -318,6 +330,8 @@ def test_basic_consolidation_with_insufficient_preexisting_churn(spec, state): @spec_test @single_phase def test_basic_consolidation_with_compounding_credentials(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -363,6 +377,8 @@ def test_basic_consolidation_with_compounding_credentials(spec, state): @spec_test @single_phase def test_consolidation_churn_limit_balance(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -411,6 +427,8 @@ def test_consolidation_churn_limit_balance(spec, state): @spec_test @single_phase def test_consolidation_balance_larger_than_churn_limit(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -458,6 +476,8 @@ def test_consolidation_balance_larger_than_churn_limit(spec, state): @spec_test @single_phase def test_consolidation_balance_through_two_churn_epochs(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # This state has 256 validators each with 32 ETH in MINIMAL preset, 128 ETH consolidation churn current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -498,6 +518,8 @@ def test_consolidation_balance_through_two_churn_epochs(spec, state): @with_electra_and_later @spec_state_test def test_basic_switch_to_compounding(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -583,6 +605,9 @@ def test_switch_to_compounding_with_pending_consolidations_at_limit(spec, state) @spec_test @single_phase def test_incorrect_exceed_pending_consolidations_limit(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + state.pending_consolidations = [ spec.PendingConsolidation(source_index=0, target_index=1) ] * spec.PENDING_CONSOLIDATIONS_LIMIT @@ -614,6 +639,9 @@ def test_incorrect_exceed_pending_consolidations_limit(spec, state): @spec_state_test @single_phase def test_incorrect_not_enough_consolidation_churn_available(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + state.pending_consolidations = [ spec.PendingConsolidation(source_index=0, target_index=1) ] @@ -651,6 +679,8 @@ def test_incorrect_not_enough_consolidation_churn_available(spec, state): @spec_test @single_phase def test_incorrect_exited_source(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -686,6 +716,8 @@ def test_incorrect_exited_source(spec, state): @spec_test @single_phase def test_incorrect_exited_target(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -720,6 +752,8 @@ def test_incorrect_exited_target(spec, state): @spec_test @single_phase def test_incorrect_inactive_source(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -755,6 +789,8 @@ def test_incorrect_inactive_source(spec, state): @spec_test @single_phase def test_incorrect_inactive_target(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -790,6 +826,8 @@ def test_incorrect_inactive_target(spec, state): @spec_test @single_phase def test_incorrect_no_source_execution_withdrawal_credential(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up a correct consolidation, but source does not have # an execution withdrawal credential current_epoch = spec.get_current_epoch(state) @@ -820,6 +858,8 @@ def test_incorrect_no_source_execution_withdrawal_credential(spec, state): @spec_test @single_phase def test_incorrect_no_target_execution_withdrawal_credential(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up a correct consolidation, but target does not have # an execution withdrawal credential current_epoch = spec.get_current_epoch(state) @@ -852,6 +892,8 @@ def test_incorrect_no_target_execution_withdrawal_credential(spec, state): @spec_test @single_phase def test_incorrect_incorrect_source_address(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -885,6 +927,8 @@ def test_incorrect_incorrect_source_address(spec, state): @spec_test @single_phase def test_incorrect_unknown_source_pubkey(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -918,6 +962,8 @@ def test_incorrect_unknown_source_pubkey(spec, state): @spec_test @single_phase def test_incorrect_unknown_target_pubkey(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH # Set up an otherwise correct consolidation current_epoch = spec.get_current_epoch(state) source_index = spec.get_active_validator_indices(state, current_epoch)[0] @@ -942,6 +988,80 @@ def test_incorrect_unknown_target_pubkey(spec, state): ) +@with_electra_and_later +@with_presets([MINIMAL], "need sufficient consolidation churn limit") +@with_custom_state( + balances_fn=scaled_churn_balances_exceed_activation_exit_churn_limit, + threshold_fn=default_activation_threshold, +) +@spec_test +@single_phase +def test_incorrect_source_has_pending_withdrawal(spec, state): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for consolidation + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + # Set up an otherwise correct consolidation + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + source_address = b"\x22" * 20 + excess_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 4 + set_eth1_withdrawal_credential_with_balance( + spec, state, source_index, address=source_address, balance=spec.MIN_ACTIVATION_BALANCE + excess_balance + ) + consolidation = spec.ConsolidationRequest( + source_address=source_address, + source_pubkey=state.validators[source_index].pubkey, + target_pubkey=state.validators[target_index].pubkey, + ) + set_eth1_withdrawal_credential_with_balance(spec, state, target_index) + + # Create pending withdrawal + pending_withdrawal = spec.PendingPartialWithdrawal( + index=0, amount=excess_balance, withdrawable_epoch=current_epoch + ) + state.pending_partial_withdrawals.append(pending_withdrawal) + + # Check the return condition + assert spec.get_pending_balance_to_withdraw(state, source_index) > 0 + + yield from run_consolidation_processing( + spec, state, consolidation, success=False + ) + + +@with_electra_and_later +@with_presets([MINIMAL], "need sufficient consolidation churn limit") +@with_custom_state( + balances_fn=scaled_churn_balances_exceed_activation_exit_churn_limit, + threshold_fn=default_activation_threshold, +) +@spec_test +@single_phase +def test_incorrect_source_not_active_long_enough(spec, state): + # Set up an otherwise correct consolidation + current_epoch = spec.get_current_epoch(state) + source_index = spec.get_active_validator_indices(state, current_epoch)[0] + target_index = spec.get_active_validator_indices(state, current_epoch)[1] + source_address = b"\x22" * 20 + excess_balance = spec.EFFECTIVE_BALANCE_INCREMENT // 4 + set_eth1_withdrawal_credential_with_balance( + spec, state, source_index, address=source_address, balance=spec.MIN_ACTIVATION_BALANCE + excess_balance + ) + consolidation = spec.ConsolidationRequest( + source_address=source_address, + source_pubkey=state.validators[source_index].pubkey, + target_pubkey=state.validators[target_index].pubkey, + ) + set_eth1_withdrawal_credential_with_balance(spec, state, target_index) + + # Check the return condition + assert current_epoch < state.validators[source_index].activation_epoch + spec.config.SHARD_COMMITTEE_PERIOD + + yield from run_consolidation_processing( + spec, state, consolidation, success=False + ) + + @with_electra_and_later @spec_state_test def test_switch_to_compounding_exited_source(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py index 555eae85b5..1757c79994 100644 --- a/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/electra/block_processing/test_process_withdrawals.py @@ -11,7 +11,7 @@ next_slot, ) from eth2spec.test.helpers.withdrawals import ( - prepare_expected_withdrawals_compounding, + prepare_expected_withdrawals, run_withdrawals_processing, set_compounding_withdrawal_credential_with_balance, prepare_pending_withdrawal, @@ -23,11 +23,11 @@ def test_success_mixed_fully_and_partial_withdrawable_compounding(spec, state): num_full_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 2 num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD - num_full_withdrawals - fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals_compounding( + fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( spec, state, rng=random.Random(42), - num_full_withdrawals=num_full_withdrawals, - num_partial_withdrawals_sweep=num_partial_withdrawals, + num_full_withdrawals_comp=num_full_withdrawals, + num_partial_withdrawals_comp=num_partial_withdrawals, ) next_slot(spec, state) @@ -94,14 +94,351 @@ def test_pending_withdrawals_one_skipped_one_effective(spec, state): index_0 = 3 index_1 = 5 - withdrawal_0 = prepare_pending_withdrawal(spec, state, index_0) - withdrawal_1 = prepare_pending_withdrawal(spec, state, index_1) + pending_withdrawal_0 = prepare_pending_withdrawal(spec, state, index_0) + pending_withdrawal_1 = prepare_pending_withdrawal(spec, state, index_1) # If validator doesn't have an excess balance pending withdrawal is skipped state.balances[index_0] = spec.MIN_ACTIVATION_BALANCE execution_payload = build_empty_execution_payload(spec, state) - assert state.pending_partial_withdrawals == [withdrawal_0, withdrawal_1] - yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=1) + assert state.pending_partial_withdrawals == [pending_withdrawal_0, pending_withdrawal_1] + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=1, + pending_withdrawal_requests=[pending_withdrawal_1] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_next_epoch(spec, state): + validator_index = len(state.validators) // 2 + next_epoch = spec.get_current_epoch(state) + 1 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index, withdrawable_epoch=next_epoch) + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [pending_withdrawal] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_at_max(spec, state): + pending_withdrawal_requests = [] + # Create spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1 partial withdrawals + for i in range(0, spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1): + pending_withdrawal = prepare_pending_withdrawal(spec, state, i) + pending_withdrawal_requests.append(pending_withdrawal) + + assert len(state.pending_partial_withdrawals) == spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1 + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, + pending_withdrawal_requests=pending_withdrawal_requests[:spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP] + ) + + withdrawals_exceeding_max = pending_withdrawal_requests[spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP:] + assert state.pending_partial_withdrawals == withdrawals_exceeding_max + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_exiting_validator(spec, state): + validator_index = len(state.validators) // 2 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index) + spec.initiate_validator_exit(state, pending_withdrawal.index) + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_low_effective_balance(spec, state): + validator_index = len(state.validators) // 2 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index) + state.validators[pending_withdrawal.index].effective_balance = ( + spec.MIN_ACTIVATION_BALANCE - spec.EFFECTIVE_BALANCE_INCREMENT + ) + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_no_excess_balance(spec, state): + validator_index = len(state.validators) // 2 + + pending_withdrawal = prepare_pending_withdrawal(spec, state, validator_index) + state.balances[pending_withdrawal.index] = spec.MIN_ACTIVATION_BALANCE + + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=0) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_ineffective_sweep_on_top(spec, state): + # Ensure validator will be processed by the sweep + validator_index = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + pending_withdrawal = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + ) + + # Check that validator is partially withdrawable before pending withdrawal is processed + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] + ) + # And is not partially withdrawable thereafter + assert not spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal.amount + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=1, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[], + pending_withdrawal_requests=[pending_withdrawal] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_ineffective_sweep_on_top_2(spec, state): + # Ensure validator will be processed by the sweep + validator_index = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + pending_withdrawal_0 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT // 2 + ) + + pending_withdrawal_1 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT + ) + + # Set excess balance in a way that validator + # becomes not partially withdrawable only after the second pending withdrawal is processed + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal_0.amount + ) + assert not spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal_0.amount - pending_withdrawal_1.amount + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=2, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[], + pending_withdrawal_requests=[pending_withdrawal_0, pending_withdrawal_1] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_effective_sweep_on_top(spec, state): + # Ensure validator will be processed by the sweep + validator_index = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + pending_withdrawal_0 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT // 2 + ) + + pending_withdrawal_1 = prepare_pending_withdrawal( + spec, state, + validator_index, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT + ) + + # Set excess balance to requested amount times three, + # so the validator is partially withdrawable after pending withdrawal is processed + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT * 2 + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index], + state.balances[validator_index] - pending_withdrawal_0.amount - pending_withdrawal_1.amount + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=3, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[validator_index], + pending_withdrawal_requests=[pending_withdrawal_0, pending_withdrawal_1] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_with_sweep_different_validator(spec, state): + # Ensure validator will be processed by the sweep + validator_index_0 = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 - 1 + validator_index_1 = min(len(state.validators), spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP) // 2 + + # Initiate pending withdrawal for the first validator + pending_withdrawal_0 = prepare_pending_withdrawal( + spec, state, + validator_index_0, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + amount=spec.EFFECTIVE_BALANCE_INCREMENT + ) + + # Make the second validator partially withdrawable by the sweep + set_compounding_withdrawal_credential_with_balance( + spec, state, validator_index_1, + effective_balance=spec.MAX_EFFECTIVE_BALANCE_ELECTRA, + balance=(spec.MAX_EFFECTIVE_BALANCE_ELECTRA + spec.EFFECTIVE_BALANCE_INCREMENT) + ) + + assert spec.is_partially_withdrawable_validator( + state.validators[validator_index_1], + state.balances[validator_index_1] + ) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=2, + fully_withdrawable_indices=[], + partial_withdrawals_indices=[validator_index_1], + pending_withdrawal_requests=[pending_withdrawal_0] + ) + + assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_mixed_with_sweep_and_fully_withdrawable(spec, state): + num_full_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_full_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_pending_withdrawal_requests = spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP // 2 + + fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( + spec, state, + rng=random.Random(42), + num_full_withdrawals=num_full_withdrawals, + num_partial_withdrawals=num_partial_withdrawals, + num_full_withdrawals_comp=num_full_withdrawals_comp, + num_partial_withdrawals_comp=num_partial_withdrawals_comp, + ) + + pending_withdrawal_requests = [] + for index in range(0, len(state.validators)): + if len(pending_withdrawal_requests) >= num_pending_withdrawal_requests: + break + if index in (fully_withdrawable_indices + partial_withdrawals_indices): + continue + + pending_withdrawal = prepare_pending_withdrawal(spec, state, index) + pending_withdrawal_requests.append(pending_withdrawal) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD, + fully_withdrawable_indices=fully_withdrawable_indices, + partial_withdrawals_indices=partial_withdrawals_indices, + pending_withdrawal_requests=pending_withdrawal_requests + ) assert state.pending_partial_withdrawals == [] + + +@with_electra_and_later +@spec_state_test +def test_pending_withdrawals_at_max_mixed_with_sweep_and_fully_withdrawable(spec, state): + num_full_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_full_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_partial_withdrawals_comp = spec.MAX_WITHDRAWALS_PER_PAYLOAD // 4 + num_pending_withdrawal_requests = spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + 1 + + fully_withdrawable_indices, partial_withdrawals_indices = prepare_expected_withdrawals( + spec, state, + rng=random.Random(42), + num_full_withdrawals=num_full_withdrawals, + num_partial_withdrawals=num_partial_withdrawals, + num_full_withdrawals_comp=num_full_withdrawals_comp, + num_partial_withdrawals_comp=num_partial_withdrawals_comp, + ) + + pending_withdrawal_requests = [] + for index in range(0, len(state.validators)): + if len(pending_withdrawal_requests) >= num_pending_withdrawal_requests: + break + if index in (fully_withdrawable_indices + partial_withdrawals_indices): + continue + + pending_withdrawal = prepare_pending_withdrawal(spec, state, index) + pending_withdrawal_requests.append(pending_withdrawal) + + next_slot(spec, state) + execution_payload = build_empty_execution_payload(spec, state) + yield from run_withdrawals_processing( + spec, state, + execution_payload, + num_expected_withdrawals=spec.MAX_WITHDRAWALS_PER_PAYLOAD, + fully_withdrawable_indices=fully_withdrawable_indices, + partial_withdrawals_indices=partial_withdrawals_indices, + pending_withdrawal_requests=pending_withdrawal_requests[:spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP] + ) + + withdrawals_exceeding_max = pending_withdrawal_requests[spec.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP:] + assert state.pending_partial_withdrawals == withdrawals_exceeding_max diff --git a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py index e569be35e3..884bfcb4eb 100644 --- a/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py +++ b/tests/core/pyspec/eth2spec/test/electra/fork/test_electra_fork_basic.py @@ -94,6 +94,30 @@ def test_fork_pre_activation(spec, phases, state): assert len(post_state.pending_deposits) > 0 +@with_phases(phases=[DENEB], other_phases=[ELECTRA]) +@spec_test +@with_state +@with_meta_tags(ELECTRA_FORK_TEST_META_TAGS) +def test_fork_pending_deposits_are_sorted(spec, phases, state): + post_spec = phases[ELECTRA] + state.validators[0].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[0].activation_eligibility_epoch = 2 + state.validators[1].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[1].activation_eligibility_epoch = 3 + state.validators[2].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[2].activation_eligibility_epoch = 2 + state.validators[3].activation_epoch = spec.FAR_FUTURE_EPOCH + state.validators[3].activation_eligibility_epoch = 1 + + post_state = yield from run_fork_test(post_spec, state) + + assert len(post_state.pending_deposits) == 4 + assert post_state.pending_deposits[0].pubkey == state.validators[3].pubkey + assert post_state.pending_deposits[1].pubkey == state.validators[0].pubkey + assert post_state.pending_deposits[2].pubkey == state.validators[2].pubkey + assert post_state.pending_deposits[3].pubkey == state.validators[1].pubkey + + @with_phases(phases=[DENEB], other_phases=[ELECTRA]) @spec_test @with_state diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/blob.py similarity index 57% rename from tests/core/pyspec/eth2spec/test/helpers/sharding.py rename to tests/core/pyspec/eth2spec/test/helpers/blob.py index 07d73e2b72..fb2354a501 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/blob.py @@ -1,57 +1,28 @@ import random -from eth2spec.utils.ssz.ssz_typing import ( - Container, - Bytes20, Bytes32, - ByteList, - List, - Union, - boolean, - uint256, uint64, - uint8, -) -from eth2spec.utils.ssz.ssz_impl import serialize - - -# -# Containers from EIP-4844 -# -MAX_CALLDATA_SIZE = 2**24 -MAX_VERSIONED_HASHES_LIST_SIZE = 2**24 -MAX_ACCESS_LIST_STORAGE_KEYS = 2**24 -MAX_ACCESS_LIST_SIZE = 2**24 - - -BLOB_TX_TYPE = uint8(0x03) - - -class AccessTuple(Container): - address: Bytes20 # Address = Bytes20 - storage_keys: List[Bytes32, MAX_ACCESS_LIST_STORAGE_KEYS] - - -class ECDSASignature(Container): - y_parity: boolean - r: uint256 - s: uint256 - - -class BlobTransaction(Container): - chain_id: uint256 - nonce: uint64 - max_priority_fee_per_gas: uint256 - max_fee_per_gas: uint256 - gas: uint64 - to: Union[None, Bytes20] # Address = Bytes20 - value: uint256 - data: ByteList[MAX_CALLDATA_SIZE] - access_list: List[AccessTuple, MAX_ACCESS_LIST_SIZE] - max_fee_per_blob_gas: uint256 - blob_versioned_hashes: List[Bytes32, MAX_VERSIONED_HASHES_LIST_SIZE] - - -class SignedBlobTransaction(Container): - message: BlobTransaction - signature: ECDSASignature +from rlp import encode, Serializable +from rlp.sedes import Binary, CountableList, List as RLPList, big_endian_int, binary + + +class Eip4844RlpTransaction(Serializable): + fields = ( + ('chain_id', big_endian_int), + ('nonce', big_endian_int), + ('max_priority_fee_per_gas', big_endian_int), + ('max_fee_per_gas', big_endian_int), + ('gas_limit', big_endian_int), + ('to', Binary(20, 20)), + ('value', big_endian_int), + ('data', binary), + ('access_list', CountableList(RLPList([ + Binary(20, 20), + CountableList(Binary(32, 32)), + ]))), + ('max_fee_per_blob_gas', big_endian_int), + ('blob_versioned_hashes', CountableList(Binary(32, 32))), + ('signature_y_parity', big_endian_int), + ('signature_r', big_endian_int), + ('signature_s', big_endian_int), + ) def get_sample_blob(spec, rng=random.Random(5566), is_valid_blob=True): @@ -91,7 +62,7 @@ def get_poly_in_both_forms(spec, rng=None): return coeffs, evals -def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): +def get_sample_blob_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): blobs = [] blob_kzg_commitments = [] blob_kzg_proofs = [] @@ -110,11 +81,21 @@ def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_b blob_kzg_proofs.append(blob_kzg_proof) blob_versioned_hashes.append(blob_versioned_hash) - signed_blob_tx = SignedBlobTransaction( - message=BlobTransaction( - blob_versioned_hashes=blob_versioned_hashes, - ) + signed_blob_tx = Eip4844RlpTransaction( + chain_id=0, + nonce=0, + max_priority_fee_per_gas=0, + max_fee_per_gas=0, + gas_limit=0, + to=bytes.fromhex("0000000000000000000000000000000000000000"), + value=0, + data=bytes.fromhex(""), + access_list=[], + max_fee_per_blob_gas=0, + blob_versioned_hashes=[bytes(h) for h in blob_versioned_hashes], + signature_y_parity=0, + signature_r=0, + signature_s=0, ) - serialized_tx = serialize(signed_blob_tx) - opaque_tx = spec.uint_to_bytes(BLOB_TX_TYPE) + serialized_tx + opaque_tx = bytes([0x03]) + encode(signed_blob_tx) return opaque_tx, blobs, blob_kzg_commitments, blob_kzg_proofs diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 7e5d0a0f4c..97ed1134e2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -57,7 +57,7 @@ ELECTRA: DENEB, # Experimental patches WHISK: CAPELLA, - EIP7594: DENEB, + EIP7594: ELECTRA, EIP7732: ELECTRA, } diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 4bad581eef..0766008b84 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -55,7 +55,7 @@ def compute_trie_root_from_indexed_data(data): t = HexaryTrie(db={}) for i, obj in enumerate(data): k = encode(i, big_endian_int) - t.set(k, obj) + t.set(k, obj) # Implicitly skipped if `obj == b''` (invalid RLP) return t.root_hash @@ -353,4 +353,4 @@ def build_state_with_execution_payload_header(spec, state, execution_payload_hea def get_random_tx(rng): - return get_random_bytes_list(rng, rng.randint(0, 1000)) + return get_random_bytes_list(rng, rng.randint(1, 1000)) diff --git a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py index 518920aeb2..b1cfaf869d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/helpers/withdrawals.py @@ -63,11 +63,21 @@ def sample_withdrawal_indices(spec, state, rng, num_full_withdrawals, num_partia def prepare_expected_withdrawals(spec, state, rng, - num_full_withdrawals=0, num_partial_withdrawals=0): + num_full_withdrawals=0, num_partial_withdrawals=0, + num_full_withdrawals_comp=0, num_partial_withdrawals_comp=0): fully_withdrawable_indices, partial_withdrawals_indices = sample_withdrawal_indices( - spec, state, rng, num_full_withdrawals, num_partial_withdrawals + spec, state, rng, + num_full_withdrawals + num_full_withdrawals_comp, + num_partial_withdrawals + num_partial_withdrawals_comp ) + fully_withdrawable_indices_comp = rng.sample(fully_withdrawable_indices, num_full_withdrawals_comp) + partial_withdrawals_indices_comp = rng.sample(partial_withdrawals_indices, num_partial_withdrawals_comp) + + for index in (fully_withdrawable_indices_comp + partial_withdrawals_indices_comp): + address = state.validators[index].withdrawal_credentials[12:] + set_compounding_withdrawal_credential_with_balance(spec, state, index, address=address) + for index in fully_withdrawable_indices: set_validator_fully_withdrawable(spec, state, index) for index in partial_withdrawals_indices: @@ -97,32 +107,13 @@ def set_compounding_withdrawal_credential_with_balance(spec, state, index, state.balances[index] = balance -def prepare_expected_withdrawals_compounding(spec, state, rng, - num_full_withdrawals=0, - num_partial_withdrawals_sweep=0, - excess_balance=1000000000): - assert is_post_electra(spec) - - fully_withdrawable_indices, partial_withdrawals_sweep_indices = sample_withdrawal_indices( - spec, state, rng, num_full_withdrawals, num_partial_withdrawals_sweep - ) - - for index in fully_withdrawable_indices + partial_withdrawals_sweep_indices: - address = state.validators[index].withdrawal_credentials[12:] - set_compounding_withdrawal_credential_with_balance(spec, state, index, address=address) - - for index in fully_withdrawable_indices: - set_validator_fully_withdrawable(spec, state, index) - for index in partial_withdrawals_sweep_indices: - set_validator_partially_withdrawable(spec, state, index) - - return fully_withdrawable_indices, partial_withdrawals_sweep_indices - - def prepare_pending_withdrawal(spec, state, validator_index, - effective_balance=32_000_000_000, amount=1_000_000_000): + effective_balance=32_000_000_000, amount=1_000_000_000, withdrawable_epoch=None): assert is_post_electra(spec) + if withdrawable_epoch is None: + withdrawable_epoch = spec.get_current_epoch(state) + balance = effective_balance + amount set_compounding_withdrawal_credential_with_balance( spec, state, validator_index, effective_balance, balance @@ -131,7 +122,7 @@ def prepare_pending_withdrawal(spec, state, validator_index, withdrawal = spec.PendingPartialWithdrawal( index=validator_index, amount=amount, - withdrawable_epoch=spec.get_current_epoch(state), + withdrawable_epoch=withdrawable_epoch, ) state.pending_partial_withdrawals.append(withdrawal) @@ -175,7 +166,8 @@ def verify_post_state(state, spec, expected_withdrawals, def run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=None, - fully_withdrawable_indices=None, partial_withdrawals_indices=None, valid=True): + fully_withdrawable_indices=None, partial_withdrawals_indices=None, + pending_withdrawal_requests=None, valid=True): """ Run ``process_withdrawals``, yielding: - pre-state ('pre') @@ -206,6 +198,11 @@ def run_withdrawals_processing(spec, state, execution_payload, num_expected_with yield 'post', state + # Check withdrawal indices + assert state.next_withdrawal_index == pre_state.next_withdrawal_index + len(expected_withdrawals) + for index, withdrawal in enumerate(execution_payload.withdrawals): + assert withdrawal.index == pre_state.next_withdrawal_index + index + if len(expected_withdrawals) == 0: next_withdrawal_validator_index = ( pre_state.next_withdrawal_validator_index + spec.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP @@ -220,4 +217,12 @@ def run_withdrawals_processing(spec, state, execution_payload, num_expected_with if fully_withdrawable_indices is not None or partial_withdrawals_indices is not None: verify_post_state(state, spec, expected_withdrawals, fully_withdrawable_indices, partial_withdrawals_indices) + # Check withdrawal requests + if pending_withdrawal_requests is not None: + assert len(pending_withdrawal_requests) <= len(execution_payload.withdrawals) + for index, request in enumerate(pending_withdrawal_requests): + withdrawal = execution_payload.withdrawals[index] + assert withdrawal.validator_index == request.index + assert withdrawal.amount == request.amount + return expected_withdrawals diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py index 78802a6ddd..6e6cdd7683 100644 --- a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -24,8 +24,8 @@ randomize_state as randomize_state_helper, patch_state_to_non_leaking, ) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, +from eth2spec.test.helpers.blob import ( + get_sample_blob_tx, ) from eth2spec.test.helpers.state import ( next_slot, @@ -250,7 +250,7 @@ def random_block_capella(spec, state, signed_blocks, scenario_state, rng=Random( def random_block_deneb(spec, state, signed_blocks, scenario_state, rng=Random(3456)): block = random_block_capella(spec, state, signed_blocks, scenario_state, rng=rng) # TODO: more commitments. blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx( + opaque_tx, _, blob_kzg_commitments, _ = get_sample_blob_tx( spec, blob_count=rng.randint(0, spec.config.MAX_BLOBS_PER_BLOCK), rng=rng) block.body.execution_payload.transactions.append(opaque_tx) block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload, state) diff --git a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py index 65808038ea..645cbff775 100644 --- a/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/tests/core/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,14 +1,26 @@ from typing import TypeVar from remerkleable.basic import uint -from remerkleable.core import View +from remerkleable.core import Type, View from remerkleable.byte_arrays import Bytes32 -def serialize(obj: View) -> bytes: +def ssz_serialize(obj: View) -> bytes: return obj.encode_bytes() +def serialize(obj: View) -> bytes: + return ssz_serialize(obj) + + +def ssz_deserialize(typ: Type[View], data: bytes) -> View: + return typ.decode_bytes(data) + + +def deserialize(typ: Type[View], data: bytes) -> View: + return ssz_deserialize(typ, data) + + def hash_tree_root(obj: View) -> Bytes32: return Bytes32(obj.get_backing().merkle_root()) diff --git a/tests/formats/kzg_7594/README.md b/tests/formats/kzg_7594/README.md index f03bc3707c..baa8589f8a 100644 --- a/tests/formats/kzg_7594/README.md +++ b/tests/formats/kzg_7594/README.md @@ -6,7 +6,6 @@ We do not recommend rolling your own crypto or using an untested KZG library. The KZG test suite runner has the following handlers: -- [`compute_cells`](./compute_cells.md) - [`compute_cells_and_kzg_proofs`](./compute_cells_and_kzg_proofs.md) +- [`recover_cells_and_kzg_proofs`](./recover_cells_and_kzg_proofs.md) - [`verify_cell_kzg_proof_batch`](./verify_cell_kzg_proof_batch.md) -- [`recover_all_cells`](./recover_all_cells.md) diff --git a/tests/generators/README.md b/tests/generators/README.md index 0146ca35e8..148415f4de 100644 --- a/tests/generators/README.md +++ b/tests/generators/README.md @@ -185,6 +185,7 @@ if __name__ == "__main__": PHASE0: phase_0_mods, ALTAIR: altair_mods, } + check_mods(all_mods, "sanity") run_state_test_generators(runner_name="sanity", all_mods=all_mods) ``` diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 1de3e84e3d..4c848a9d12 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -63,5 +63,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "epoch_processing") run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 6d33ae6bdc..b31e949421 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -18,5 +18,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "finality") run_state_test_generators(runner_name="finality", all_mods=all_mods) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 92c67e4c0b..10a52fb954 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -37,5 +37,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "fork_choice") run_state_test_generators(runner_name="fork_choice", all_mods=all_mods) diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 57b680ebac..9e82d1c112 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -26,5 +26,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "genesis") run_state_test_generators(runner_name="genesis", all_mods=all_mods) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index a3cdfd62fd..23aed84775 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA -from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators, check_mods if __name__ == "__main__": @@ -24,5 +24,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "light_client") run_state_test_generators(runner_name="light_client", all_mods=all_mods) diff --git a/tests/generators/merkle_proof/main.py b/tests/generators/merkle_proof/main.py index 69500137ab..9ae985b865 100644 --- a/tests/generators/merkle_proof/main.py +++ b/tests/generators/merkle_proof/main.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import DENEB, ELECTRA, EIP7594 -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods if __name__ == "__main__": @@ -10,12 +10,13 @@ 'single_merkle_proof', ]} electra_mods = deneb_mods - eip_7594_mods = combine_mods(_new_eip7594_mods, deneb_mods) + eip_7594_mods = combine_mods(_new_eip7594_mods, electra_mods) all_mods = { DENEB: deneb_mods, ELECTRA: electra_mods, EIP7594: eip_7594_mods, } + check_mods(all_mods, "merkle_proof") run_state_test_generators(runner_name="merkle_proof", all_mods=all_mods) diff --git a/tests/generators/networking/main.py b/tests/generators/networking/main.py index 2681daf68b..52b94929f7 100644 --- a/tests/generators/networking/main.py +++ b/tests/generators/networking/main.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.constants import EIP7594 -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods if __name__ == "__main__": @@ -10,5 +10,6 @@ all_mods = { EIP7594: eip7594_mods } + check_mods(all_mods, "networking") run_state_test_generators(runner_name="networking", all_mods=all_mods) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index ae66843f61..9e3a7c21a4 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -61,5 +61,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "block_processing") run_state_test_generators(runner_name="operations", all_mods=all_mods) diff --git a/tests/generators/random/main.py b/tests/generators/random/main.py index 1c9472b5ad..1d176c03ca 100644 --- a/tests/generators/random/main.py +++ b/tests/generators/random/main.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA, ) -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods if __name__ == "__main__": @@ -32,5 +32,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "random") run_state_test_generators(runner_name="random", all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index dcf2c6ce23..f1b27133a9 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -27,5 +27,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "rewards") run_state_test_generators(runner_name="rewards", all_mods=all_mods) diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 7b1eff0324..8039b82a44 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,5 +1,5 @@ from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, ELECTRA -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods, check_mods if __name__ == "__main__": @@ -44,5 +44,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "sanity") run_state_test_generators(runner_name="sanity", all_mods=all_mods) diff --git a/tests/generators/sync/main.py b/tests/generators/sync/main.py index 94e392b775..fd2f3f0209 100644 --- a/tests/generators/sync/main.py +++ b/tests/generators/sync/main.py @@ -1,4 +1,4 @@ -from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, check_mods from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, DENEB, ELECTRA @@ -16,5 +16,6 @@ DENEB: deneb_mods, ELECTRA: electra_mods, } + check_mods(all_mods, "sync") run_state_test_generators(runner_name="sync", all_mods=all_mods)