From 53e528e56c3ed3a8d7c8074a380023e275a0c2e9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 00:28:20 +0800 Subject: [PATCH 01/61] work in progress: setup test libs, pyspec, pytests --- .gitignore | 6 +- {scripts => py_tests}/__init__.py | 0 {tests => py_tests}/conftest.py | 0 {tests => py_tests/phase0}/__init__.py | 0 .../test_process_attestation.py | 0 .../test_process_block_header.py | 0 .../block_processing/test_process_deposit.py | 0 .../test_process_proposer_slashing.py | 0 .../block_processing/test_voluntary_exit.py | 0 {tests => py_tests}/phase0/conftest.py | 0 {tests => py_tests}/phase0/helpers.py | 0 {tests => py_tests}/phase0/test_sanity.py | 0 requirements.txt => py_tests/requirements.txt | 0 test_libs/gen_helpers/README.txt | 4 + .../gen_helpers/gen_base}/__init__.py | 0 test_libs/gen_helpers/gen_base/gen_runner.py | 81 +++++++++++++++++++ test_libs/gen_helpers/gen_base/gen_suite.py | 17 ++++ test_libs/gen_helpers/gen_base/gen_typing.py | 5 ++ test_libs/gen_helpers/requirements.txt | 2 + test_libs/gen_helpers/setup.py | 8 ++ test_libs/pyspec/README.md | 3 + {utils => test_libs/pyspec/debug}/__init__.py | 0 .../pyspec/debug}/jsonize.py | 2 +- .../pyspec}/phase0/__init__.py | 0 .../pyspec}/phase0/state_transition.py | 0 test_libs/pyspec/requirements.txt | 4 + test_libs/pyspec/setup.py | 13 +++ test_libs/pyspec/utils/__init__.py | 0 .../pyspec/utils}/bls_stub.py | 0 .../pyspec/utils}/hash_function.py | 0 .../pyspec/utils}/merkle_minimal.py | 0 .../pyspec/utils}/minimal_ssz.py | 0 32 files changed, 143 insertions(+), 2 deletions(-) rename {scripts => py_tests}/__init__.py (100%) rename {tests => py_tests}/conftest.py (100%) rename {tests => py_tests/phase0}/__init__.py (100%) rename {tests => py_tests}/phase0/block_processing/test_process_attestation.py (100%) rename {tests => py_tests}/phase0/block_processing/test_process_block_header.py (100%) rename {tests => py_tests}/phase0/block_processing/test_process_deposit.py (100%) rename {tests => py_tests}/phase0/block_processing/test_process_proposer_slashing.py (100%) rename {tests => py_tests}/phase0/block_processing/test_voluntary_exit.py (100%) rename {tests => py_tests}/phase0/conftest.py (100%) rename {tests => py_tests}/phase0/helpers.py (100%) rename {tests => py_tests}/phase0/test_sanity.py (100%) rename requirements.txt => py_tests/requirements.txt (100%) create mode 100644 test_libs/gen_helpers/README.txt rename {tests/phase0 => test_libs/gen_helpers/gen_base}/__init__.py (100%) create mode 100644 test_libs/gen_helpers/gen_base/gen_runner.py create mode 100644 test_libs/gen_helpers/gen_base/gen_suite.py create mode 100644 test_libs/gen_helpers/gen_base/gen_typing.py create mode 100644 test_libs/gen_helpers/requirements.txt create mode 100644 test_libs/gen_helpers/setup.py create mode 100644 test_libs/pyspec/README.md rename {utils => test_libs/pyspec/debug}/__init__.py (100%) rename {utils/phase0 => test_libs/pyspec/debug}/jsonize.py (97%) rename {utils => test_libs/pyspec}/phase0/__init__.py (100%) rename {utils => test_libs/pyspec}/phase0/state_transition.py (100%) create mode 100644 test_libs/pyspec/requirements.txt create mode 100644 test_libs/pyspec/setup.py create mode 100644 test_libs/pyspec/utils/__init__.py rename {utils/phase0 => test_libs/pyspec/utils}/bls_stub.py (100%) rename {utils/phase0 => test_libs/pyspec/utils}/hash_function.py (100%) rename {utils/phase0 => test_libs/pyspec/utils}/merkle_minimal.py (100%) rename {utils/phase0 => test_libs/pyspec/utils}/minimal_ssz.py (100%) diff --git a/.gitignore b/.gitignore index f33dd5256e..816ecfa269 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ *.pyc /__pycache__ -/venv +venv +.venvs +.venv /.pytest_cache build/ output/ + +yaml_tests/ diff --git a/scripts/__init__.py b/py_tests/__init__.py similarity index 100% rename from scripts/__init__.py rename to py_tests/__init__.py diff --git a/tests/conftest.py b/py_tests/conftest.py similarity index 100% rename from tests/conftest.py rename to py_tests/conftest.py diff --git a/tests/__init__.py b/py_tests/phase0/__init__.py similarity index 100% rename from tests/__init__.py rename to py_tests/phase0/__init__.py diff --git a/tests/phase0/block_processing/test_process_attestation.py b/py_tests/phase0/block_processing/test_process_attestation.py similarity index 100% rename from tests/phase0/block_processing/test_process_attestation.py rename to py_tests/phase0/block_processing/test_process_attestation.py diff --git a/tests/phase0/block_processing/test_process_block_header.py b/py_tests/phase0/block_processing/test_process_block_header.py similarity index 100% rename from tests/phase0/block_processing/test_process_block_header.py rename to py_tests/phase0/block_processing/test_process_block_header.py diff --git a/tests/phase0/block_processing/test_process_deposit.py b/py_tests/phase0/block_processing/test_process_deposit.py similarity index 100% rename from tests/phase0/block_processing/test_process_deposit.py rename to py_tests/phase0/block_processing/test_process_deposit.py diff --git a/tests/phase0/block_processing/test_process_proposer_slashing.py b/py_tests/phase0/block_processing/test_process_proposer_slashing.py similarity index 100% rename from tests/phase0/block_processing/test_process_proposer_slashing.py rename to py_tests/phase0/block_processing/test_process_proposer_slashing.py diff --git a/tests/phase0/block_processing/test_voluntary_exit.py b/py_tests/phase0/block_processing/test_voluntary_exit.py similarity index 100% rename from tests/phase0/block_processing/test_voluntary_exit.py rename to py_tests/phase0/block_processing/test_voluntary_exit.py diff --git a/tests/phase0/conftest.py b/py_tests/phase0/conftest.py similarity index 100% rename from tests/phase0/conftest.py rename to py_tests/phase0/conftest.py diff --git a/tests/phase0/helpers.py b/py_tests/phase0/helpers.py similarity index 100% rename from tests/phase0/helpers.py rename to py_tests/phase0/helpers.py diff --git a/tests/phase0/test_sanity.py b/py_tests/phase0/test_sanity.py similarity index 100% rename from tests/phase0/test_sanity.py rename to py_tests/phase0/test_sanity.py diff --git a/requirements.txt b/py_tests/requirements.txt similarity index 100% rename from requirements.txt rename to py_tests/requirements.txt diff --git a/test_libs/gen_helpers/README.txt b/test_libs/gen_helpers/README.txt new file mode 100644 index 0000000000..10c97f2d69 --- /dev/null +++ b/test_libs/gen_helpers/README.txt @@ -0,0 +1,4 @@ +ETH 2.0 test generator helpers + +`eth2_test_gen_base`: A util to quickly write new test suite generators with. + diff --git a/tests/phase0/__init__.py b/test_libs/gen_helpers/gen_base/__init__.py similarity index 100% rename from tests/phase0/__init__.py rename to test_libs/gen_helpers/gen_base/__init__.py diff --git a/test_libs/gen_helpers/gen_base/gen_runner.py b/test_libs/gen_helpers/gen_base/gen_runner.py new file mode 100644 index 0000000000..ad729449ac --- /dev/null +++ b/test_libs/gen_helpers/gen_base/gen_runner.py @@ -0,0 +1,81 @@ +import argparse +import pathlib +import sys +from typing import List + +from ruamel.yaml import ( + YAML, +) + +from gen_base.gen_typing import TestSuiteCreator + + +def make_filename_for_test(test): + title = test["title"] + filename = title.lower().replace(" ", "_") + ".yaml" + return pathlib.Path(filename) + + +def validate_output_dir(path_str): + path = pathlib.Path(path_str) + + if not path.exists(): + raise argparse.ArgumentTypeError("Output directory must exist") + + if not path.is_dir(): + raise argparse.ArgumentTypeError("Output path must lead to a directory") + + return path + + +def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): + """ + Implementation for a general test generator. + :param generator_name: The name of the generator. (lowercase snake_case) + :param suite_creators: A list of suite creators, each of these builds a list of test cases. + :return: + """ + + parser = argparse.ArgumentParser( + prog="gen-" + generator_name, + description=f"Generate YAML test suite files for {generator_name}", + ) + parser.add_argument( + "-o", + "--output-dir", + dest="output_dir", + required=True, + type=validate_output_dir, + help="directory into which the generated YAML files will be dumped" + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="if set overwrite test files if they exist", + ) + + args = parser.parse_args() + output_dir = args.output_dir + if not args.force: + file_mode = "x" + else: + file_mode = "w" + + yaml = YAML(pure=True) + + print(f"Generating tests for {generator_name}, creating {len(suite_creators)} test suite files...") + for suite_creator in suite_creators: + suite = suite_creator() + + filename = make_filename_for_test(suite) + path = output_dir / filename + + try: + with path.open(file_mode) as f: + yaml.dump(suite, f) + except IOError as e: + sys.exit(f'Error when dumping test "{suite["title"]}" ({e})') + + print("done.") diff --git a/test_libs/gen_helpers/gen_base/gen_suite.py b/test_libs/gen_helpers/gen_base/gen_suite.py new file mode 100644 index 0000000000..fdfac8292c --- /dev/null +++ b/test_libs/gen_helpers/gen_base/gen_suite.py @@ -0,0 +1,17 @@ +from typing import Iterable + +from eth_utils import ( + to_dict, +) + +from gen_base.gen_typing import TestCase + + +@to_dict +def render_suite(*, title: str, summary: str, fork: str, config: str, test_cases: Iterable[TestCase]): + yield "title", title + if summary is not None: + yield "summary", summary + yield "fork", fork + yield "config", config + yield "test_cases", test_cases diff --git a/test_libs/gen_helpers/gen_base/gen_typing.py b/test_libs/gen_helpers/gen_base/gen_typing.py new file mode 100644 index 0000000000..1384c870ff --- /dev/null +++ b/test_libs/gen_helpers/gen_base/gen_typing.py @@ -0,0 +1,5 @@ +from typing import Callable, Dict, Any + +TestCase = Dict[str, Any] +TestSuite = Dict[str, Any] +TestSuiteCreator = Callable[[], TestSuite] diff --git a/test_libs/gen_helpers/requirements.txt b/test_libs/gen_helpers/requirements.txt new file mode 100644 index 0000000000..3d6a39458e --- /dev/null +++ b/test_libs/gen_helpers/requirements.txt @@ -0,0 +1,2 @@ +ruamel.yaml==0.15.87 +eth-utils==1.4.1 diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py new file mode 100644 index 0000000000..a6c65c2124 --- /dev/null +++ b/test_libs/gen_helpers/setup.py @@ -0,0 +1,8 @@ +from distutils.core import setup + +setup( + name='gen_helpers', + version='1.0', + packages=['gen_base'], + install_requires=['ruamel.yaml', 'eth-utils'] +) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md new file mode 100644 index 0000000000..2747ad8f31 --- /dev/null +++ b/test_libs/pyspec/README.md @@ -0,0 +1,3 @@ +# ETH 2.0 PySpec + +The py \ No newline at end of file diff --git a/utils/__init__.py b/test_libs/pyspec/debug/__init__.py similarity index 100% rename from utils/__init__.py rename to test_libs/pyspec/debug/__init__.py diff --git a/utils/phase0/jsonize.py b/test_libs/pyspec/debug/jsonize.py similarity index 97% rename from utils/phase0/jsonize.py rename to test_libs/pyspec/debug/jsonize.py index 816192ec6e..ac0243a5d5 100644 --- a/utils/phase0/jsonize.py +++ b/test_libs/pyspec/debug/jsonize.py @@ -1,4 +1,4 @@ -from .minimal_ssz import hash_tree_root +from shared_eth2.minimal_ssz import hash_tree_root def jsonize(value, typ, include_hash_tree_roots=False): diff --git a/utils/phase0/__init__.py b/test_libs/pyspec/phase0/__init__.py similarity index 100% rename from utils/phase0/__init__.py rename to test_libs/pyspec/phase0/__init__.py diff --git a/utils/phase0/state_transition.py b/test_libs/pyspec/phase0/state_transition.py similarity index 100% rename from utils/phase0/state_transition.py rename to test_libs/pyspec/phase0/state_transition.py diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt new file mode 100644 index 0000000000..78d41708dc --- /dev/null +++ b/test_libs/pyspec/requirements.txt @@ -0,0 +1,4 @@ +eth-utils>=1.3.0,<2 +eth-typing>=2.1.0,<3.0.0 +pycryptodome==3.7.3 +py_ecc>=1.6.0 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py new file mode 100644 index 0000000000..5d121a2633 --- /dev/null +++ b/test_libs/pyspec/setup.py @@ -0,0 +1,13 @@ +from distutils.core import setup + +setup( + name='pyspec', + version='1.0', + packages=['debug', 'utils', 'phase0'], + install_requires=[ + "eth-utils>=1.3.0,<2", + "eth-typing>=2.1.0,<3.0.0", + "pycryptodome==3.7.3", + "py_ecc>=1.6.0", + ] +) diff --git a/test_libs/pyspec/utils/__init__.py b/test_libs/pyspec/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/phase0/bls_stub.py b/test_libs/pyspec/utils/bls_stub.py similarity index 100% rename from utils/phase0/bls_stub.py rename to test_libs/pyspec/utils/bls_stub.py diff --git a/utils/phase0/hash_function.py b/test_libs/pyspec/utils/hash_function.py similarity index 100% rename from utils/phase0/hash_function.py rename to test_libs/pyspec/utils/hash_function.py diff --git a/utils/phase0/merkle_minimal.py b/test_libs/pyspec/utils/merkle_minimal.py similarity index 100% rename from utils/phase0/merkle_minimal.py rename to test_libs/pyspec/utils/merkle_minimal.py diff --git a/utils/phase0/minimal_ssz.py b/test_libs/pyspec/utils/minimal_ssz.py similarity index 100% rename from utils/phase0/minimal_ssz.py rename to test_libs/pyspec/utils/minimal_ssz.py From 64f012b276958c06e8ce7fc6ee33a4b24f260530 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 00:32:13 +0800 Subject: [PATCH 02/61] Move test-generators to specs repo Co-authored-by: Chih Cheng Liang Co-authored-by: Danny Ryan Co-authored-by: Dmitrii Shmatko Co-authored-by: Jannik Luhn Co-authored-by: Paul Hauner Co-authored-by: protolambda --- test_generators/README.md | 151 +++++++++++++++ test_generators/bls/README.md | 20 ++ test_generators/bls/main.py | 192 ++++++++++++++++++++ test_generators/bls/requirements.txt | 2 + test_generators/shuffling/README.md | 16 ++ test_generators/shuffling/constants.py | 6 + test_generators/shuffling/core_helpers.py | 95 ++++++++++ test_generators/shuffling/main.py | 160 ++++++++++++++++ test_generators/shuffling/requirements.txt | 4 + test_generators/shuffling/utils.py | 6 + test_generators/shuffling/yaml_objects.py | 25 +++ test_generators/ssz/__init__.py | 0 test_generators/ssz/main.py | 84 +++++++++ test_generators/ssz/renderers.py | 102 +++++++++++ test_generators/ssz/requirements.txt | 2 + test_generators/ssz/uint_test_generators.py | 132 ++++++++++++++ 16 files changed, 997 insertions(+) create mode 100644 test_generators/README.md create mode 100644 test_generators/bls/README.md create mode 100644 test_generators/bls/main.py create mode 100644 test_generators/bls/requirements.txt create mode 100644 test_generators/shuffling/README.md create mode 100644 test_generators/shuffling/constants.py create mode 100644 test_generators/shuffling/core_helpers.py create mode 100644 test_generators/shuffling/main.py create mode 100644 test_generators/shuffling/requirements.txt create mode 100644 test_generators/shuffling/utils.py create mode 100644 test_generators/shuffling/yaml_objects.py create mode 100644 test_generators/ssz/__init__.py create mode 100644 test_generators/ssz/main.py create mode 100644 test_generators/ssz/renderers.py create mode 100644 test_generators/ssz/requirements.txt create mode 100644 test_generators/ssz/uint_test_generators.py diff --git a/test_generators/README.md b/test_generators/README.md new file mode 100644 index 0000000000..2d6160c03c --- /dev/null +++ b/test_generators/README.md @@ -0,0 +1,151 @@ +# Eth2.0 Test Generators + +This directory of contains all the generators for YAML tests, consumed by Eth 2.0 client implementations. + +Any issues with the generators and/or generated tests should be filed + in the repository that hosts the generator outputs, here: [ethereum/eth2.0-tests](https://github.com/ethereum/eth2.0-tests/). + +Whenever a release is made, the new tests are automatically built and +[eth2TestGenBot](https://github.com/eth2TestGenBot) commits the changes to the test repository. + +## How to run generators + +pre-requisites: +- Python 3 installed +- PIP 3 +- GNU make + +### Cleaning + +This removes the existing virtual environments (`/test_generators/.venvs/`), and generated tests (`/yaml_tests/`). + +```bash +make clean +``` + +### Running all test generators + +This runs all the generators. + +```bash +make all +``` + +### Running a single generator + +The make file auto-detects generators in the `test_generators/` directory, + and provides a tests-gen target for each generator, see example. + +```bash +make ./tests/shuffling/ +``` + +## Developing a generator + +Simply open up the generator (not all at once) of choice in your favorite IDE/editor, and run: + +```bash +# Create a virtual environment (any venv/.venv/.venvs is git-ignored) +python3 -m venv .venv +# Activate the venv, this is where dependencies are installed for the generator +. .venv/bin/activate +``` + +Now that you have a virtual environment, write your generator. +It's recommended to extend the base-generator. + +Create a `requirements.txt` in the root of your generator directory: +``` +eth-utils==1.4.1 +../test_libs/gen_helpers +``` + +Install all the necessary requirements (re-run when you add more): +```bash +pip3 install -r requirements.txt --user +``` + +And write your initial test generator, extending the base generator: + +Write a `main.py` file, here's an example: + +```python +from gen_base import gen_runner, gen_suite, gen_typing + +from eth_utils import ( + to_dict, to_tuple +) + + +@to_dict +def bar_test_case(v: int): + yield "bar_v", v + yield "bar_v_plus_1", v + 1 + yield "bar_list", list(range(v)) + + +@to_tuple +def generate_bar_test_cases(): + for i in range(10): + yield bar_test_case(i) + + +def bar_test_suite() -> gen_typing.TestSuite: + return gen_suite.render_suite( + title="bar_minimal", + summary="Minimal example suite, testing bar.", + fork="v0.5.1", + config="minimal", + test_cases=generate_bar_test_cases()) + + +if __name__ == "__main__": + gen_runner.run_generator("foo", [bar_test_suite]) + +``` + +Recommendations: +- you can have more than just 1 generator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` +- you can concatenate lists of test cases, if you don't want to split it up in suites. +- you can split your suite generators into different python files/packages, good for code organization. +- use config "minimal" for performance. But also implement a suite with the default config where necessary +- the test-generator accepts `--output` and `--force` (overwrite output) + +## How to add a new test generator + +In order to add a new test generator that builds `New Tests`: + +1. Create a new directory `new_tests`, within the `test_generators` directory. + Note that `new_tests` is also the name of the directory in which the tests will appear in the tests repository later. +2. Your generator is assumed to have a `requirements.txt` file, + with any dependencies it may need. Leave it empty if your generator has none. +3. Your generator is assumed to have a `main.py` file in its root. + By adding the base generator to your requirements, you can make a generator really easily. See docs below. +4. Your generator is called with `-o some/file/path/for_testing/can/be_anything`. + The base generator helps you handle this; you only have to define suite headers, + and a list of tests for each suite you generate. +5. Finally, add any linting or testing commands to the + [circleci config file](https://github.com/ethereum/eth2.0-test-generators/blob/master/.circleci/config.yml) + if desired to increase code quality. + +Note: you do not have to change the makefile. +However, if necessary (e.g. not using python, or mixing in other languages), submit an issue, and it can be a special case. +Do note that generators should be easy to maintain, lean, and based on the spec. + +All of this should be done in a pull request to the master branch. + +To deploy new tests to the testing repository: + +1. Create a release tag with a new version number on Github. +2. Increment either the: + - major version, to indicate a change in the general testing format + - minor version, if a new test generator has been added + - path version, in other cases. + +## How to remove a test generator + +If a test generator is not needed anymore, undo the steps described above and make a new release: + +1. remove the generator folder +2. remove the generated tests in the `eth2.0-tests` repository by opening a PR there. +3. make a new release diff --git a/test_generators/bls/README.md b/test_generators/bls/README.md new file mode 100644 index 0000000000..9ce1b2f6cc --- /dev/null +++ b/test_generators/bls/README.md @@ -0,0 +1,20 @@ +# BLS Test Generator + +Explanation of BLS12-381 type hierarchy +The base unit is bytes48 of which only 381 bits are used + +- FQ: uint381 modulo field modulus +- FQ2: (FQ, FQ) +- G2: (FQ2, FQ2, FQ2) + +## Resources + +- [Eth2.0 spec](https://github.com/ethereum/eth2.0-specs/blob/master/specs/bls_signature.md) +- [Finite Field Arithmetic](http://www.springeronline.com/sgw/cda/pageitems/document/cda_downloaddocument/0,11996,0-0-45-110359-0,00.pdf) +- Chapter 2 of [Elliptic Curve Cryptography](http://cacr.uwaterloo.ca/ecc/). Darrel Hankerson, Alfred Menezes, and Scott Vanstone +- [Zcash BLS parameters](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381) +- [Trinity implementation](https://github.com/ethereum/trinity/blob/master/eth2/_utils/bls.py) + +## Comments + +Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set). \ No newline at end of file diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py new file mode 100644 index 0000000000..3c2ab454be --- /dev/null +++ b/test_generators/bls/main.py @@ -0,0 +1,192 @@ +""" +BLS test vectors generator +Usage: + "python tgen_bls path/to/output.yml" +""" + +# Standard library +import sys +from typing import Tuple + +# Third-party +import yaml + +# Ethereum +from eth_utils import int_to_big_endian, big_endian_to_int + +# Local imports +from py_ecc import bls + + +def int_to_hex(n: int) -> str: + return '0x' + int_to_big_endian(n).hex() + + +def hex_to_int(x: str) -> int: + return int(x, 16) + + +# Note: even though a domain is only an uint64, +# To avoid issues with YAML parsers that are limited to 53-bit (JS language limit) +# It is serialized as an hex string as well. +DOMAINS = [ + 0, + 1, + 1234, + 2**32-1, + 2**64-1 +] + +MESSAGES = [ + b'\x00' * 32, + b'\x56' * 32, + b'\xab' * 32, +] + +PRIVKEYS = [ + # Curve order is 256 so private keys are 32 bytes at most. + # Also not all integers is a valid private key, so using pre-generated keys + hex_to_int('0x00000000000000000000000000000000263dbd792f5b1be47ed85f8938c0f29586af0d3ac7b977f21c278fe1462040e3'), + hex_to_int('0x0000000000000000000000000000000047b8192d77bf871b62e87859d653922725724a5c031afeabc60bcef5ff665138'), + hex_to_int('0x00000000000000000000000000000000328388aff0d4a5b7dc9205abd374e7e98f3cd9f3418edb4eafda5fb16473d216'), +] + + +def hash_message(msg: bytes, + domain: int) ->Tuple[Tuple[str, str], Tuple[str, str], Tuple[str, str]]: + """ + Hash message + Input: + - Message as bytes + - domain as uint64 + Output: + - Message hash as a G2 point + """ + return [ + [ + int_to_hex(fq2.coeffs[0]), + int_to_hex(fq2.coeffs[1]), + ] + for fq2 in bls.utils.hash_to_G2(msg, domain) + ] + + +def hash_message_compressed(msg: bytes, domain: int) -> Tuple[str, str]: + """ + Hash message + Input: + - Message as bytes + - domain as uint64 + Output: + - Message hash as a compressed G2 point + """ + z1, z2 = bls.utils.compress_G2(bls.utils.hash_to_G2(msg, domain)) + return [int_to_hex(z1), int_to_hex(z2)] + + +if __name__ == '__main__': + + # Order not preserved - https://github.com/yaml/pyyaml/issues/110 + metadata = { + 'title': 'BLS signature and aggregation tests', + 'summary': 'Test vectors for BLS signature', + 'test_suite': 'bls', + 'fork': 'phase0-0.5.0', + } + + case01_message_hash_G2_uncompressed = [] + for msg in MESSAGES: + for domain in DOMAINS: + case01_message_hash_G2_uncompressed.append({ + 'input': {'message': '0x' + msg.hex(), 'domain': int_to_hex(domain)}, + 'output': hash_message(msg, domain) + }) + + case02_message_hash_G2_compressed = [] + for msg in MESSAGES: + for domain in DOMAINS: + case02_message_hash_G2_compressed.append({ + 'input': {'message': '0x' + msg.hex(), 'domain': int_to_hex(domain)}, + 'output': hash_message_compressed(msg, domain) + }) + + case03_private_to_public_key = [] + #  Used in later cases + pubkeys = [bls.privtopub(privkey) for privkey in PRIVKEYS] + #  Used in public key aggregation + pubkeys_serial = ['0x' + pubkey.hex() for pubkey in pubkeys] + case03_private_to_public_key = [ + { + 'input': int_to_hex(privkey), + 'output': pubkey_serial, + } + for privkey, pubkey_serial in zip(PRIVKEYS, pubkeys_serial) + ] + + case04_sign_messages = [] + sigs = [] # used in verify + for privkey in PRIVKEYS: + for message in MESSAGES: + for domain in DOMAINS: + sig = bls.sign(message, privkey, domain) + case04_sign_messages.append({ + 'input': { + 'privkey': int_to_hex(privkey), + 'message': '0x' + message.hex(), + 'domain': int_to_hex(domain) + }, + 'output': '0x' + sig.hex() + }) + sigs.append(sig) + + # TODO: case05_verify_messages: Verify messages signed in case04 + # It takes too long, empty for now + + case06_aggregate_sigs = [] + for domain in DOMAINS: + for message in MESSAGES: + sigs = [] + for privkey in PRIVKEYS: + sig = bls.sign(message, privkey, domain) + sigs.append(sig) + case06_aggregate_sigs.append({ + 'input': ['0x' + sig.hex() for sig in sigs], + 'output': '0x' + bls.aggregate_signatures(sigs).hex(), + }) + + case07_aggregate_pubkeys = [ + { + 'input': pubkeys_serial, + 'output': '0x' + bls.aggregate_pubkeys(pubkeys).hex(), + } + ] + + # TODO + # Aggregate verify + + # TODO + # Proof-of-possession + + with open(sys.argv[2] + "test_bls.yml", 'w') as outfile: + # Dump at top level + yaml.dump(metadata, outfile, default_flow_style=False) + # default_flow_style will unravel "ValidatorRecord" and "committee" line, + # exploding file size + yaml.dump( + {'case01_message_hash_G2_uncompressed': case01_message_hash_G2_uncompressed}, + outfile, + ) + yaml.dump( + {'case02_message_hash_G2_compressed': case02_message_hash_G2_compressed}, + outfile, + ) + yaml.dump( + {'case03_private_to_public_key': case03_private_to_public_key}, + outfile, + ) + yaml.dump({'case04_sign_messages': case04_sign_messages}, outfile) + + # Too time consuming to generate + # yaml.dump({'case05_verify_messages': case05_verify_messages}, outfile) + yaml.dump({'case06_aggregate_sigs': case06_aggregate_sigs}, outfile) + yaml.dump({'case07_aggregate_pubkeys': case07_aggregate_pubkeys}, outfile) diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt new file mode 100644 index 0000000000..3989a3a0fc --- /dev/null +++ b/test_generators/bls/requirements.txt @@ -0,0 +1,2 @@ +py-ecc==1.6.0 +PyYAML==4.2b1 diff --git a/test_generators/shuffling/README.md b/test_generators/shuffling/README.md new file mode 100644 index 0000000000..047a1b8721 --- /dev/null +++ b/test_generators/shuffling/README.md @@ -0,0 +1,16 @@ +# Shuffling Test Generator + +``` +2018 Status Research & Development GmbH +Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). + +This work uses public domain work under CC0 from the Ethereum Foundation +https://github.com/ethereum/eth2.0-specs +``` + + +This file implements a test vectors generator for the shuffling algorithm described in the Ethereum +[specs](https://github.com/ethereum/eth2.0-specs/blob/2983e68f0305551083fac7fcf9330c1fc9da3411/specs/core/0_beacon-chain.md#get_new_shuffling) + +Utilizes 'swap or not' shuffling found in [An Enciphering Scheme Based on a Card Shuffle](https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf). +See the `Generalized domain` algorithm on page 3. diff --git a/test_generators/shuffling/constants.py b/test_generators/shuffling/constants.py new file mode 100644 index 0000000000..92862f898d --- /dev/null +++ b/test_generators/shuffling/constants.py @@ -0,0 +1,6 @@ +SLOTS_PER_EPOCH = 2**6 # 64 slots, 6.4 minutes +FAR_FUTURE_EPOCH = 2**64 - 1 # uint64 max +SHARD_COUNT = 2**10 # 1024 +TARGET_COMMITTEE_SIZE = 2**7 # 128 validators +ACTIVATION_EXIT_DELAY = 2**2 # 4 epochs +SHUFFLE_ROUND_COUNT = 90 diff --git a/test_generators/shuffling/core_helpers.py b/test_generators/shuffling/core_helpers.py new file mode 100644 index 0000000000..c424b771e5 --- /dev/null +++ b/test_generators/shuffling/core_helpers.py @@ -0,0 +1,95 @@ +from typing import Any, List, NewType + +from constants import SLOTS_PER_EPOCH, SHARD_COUNT, TARGET_COMMITTEE_SIZE, SHUFFLE_ROUND_COUNT +from utils import hash +from yaml_objects import Validator + +Epoch = NewType("Epoch", int) +ValidatorIndex = NewType("ValidatorIndex", int) +Bytes32 = NewType("Bytes32", bytes) + + +def int_to_bytes1(x): + return x.to_bytes(1, 'little') + + +def int_to_bytes4(x): + return x.to_bytes(4, 'little') + + +def bytes_to_int(data: bytes) -> int: + return int.from_bytes(data, 'little') + + +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch + + +def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: + """ + Get indices of active validators from ``validators``. + """ + return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] + + +def split(values: List[Any], split_count: int) -> List[List[Any]]: + """ + Splits ``values`` into ``split_count`` pieces. + """ + list_length = len(values) + return [ + values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] + for i in range(split_count) + ] + + +def get_epoch_committee_count(active_validator_count: int) -> int: + """ + Return the number of committees in one epoch. + """ + return max( + 1, + min( + SHARD_COUNT // SLOTS_PER_EPOCH, + active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + ) + ) * SLOTS_PER_EPOCH + + +def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: + """ + Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. + + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. + """ + for round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size + flip = (pivot - index) % list_size + position = max(index, flip) + source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index + + +def get_shuffling(seed: Bytes32, + validators: List[Validator], + epoch: Epoch) -> List[List[ValidatorIndex]]: + """ + Shuffle active validators and split into crosslink committees. + Return a list of committees (each a list of validator indices). + """ + # Shuffle active validator indices + active_validator_indices = get_active_validator_indices(validators, epoch) + length = len(active_validator_indices) + shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] + + # Split the shuffled active validator indices + return split(shuffled_indices, get_epoch_committee_count(length)) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py new file mode 100644 index 0000000000..03352944a4 --- /dev/null +++ b/test_generators/shuffling/main.py @@ -0,0 +1,160 @@ +import random +import sys +import os + +import yaml + +from constants import ACTIVATION_EXIT_DELAY, FAR_FUTURE_EPOCH +from core_helpers import get_shuffling +from yaml_objects import Validator + + +def noop(self, *args, **kw): + # Prevent !!str or !!binary tags + pass + + +yaml.emitter.Emitter.process_tag = noop + + +EPOCH = 1000 # The epoch, also a mean for the normal distribution + +# Standard deviation, around 8% validators will activate or exit within +# ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator +# shuffling +RAND_EPOCH_STD = 35 +MAX_EXIT_EPOCH = 5000 # Maximum exit_epoch for easier reading + + +def active_exited_validators_generator(): + """ + Random cases with variety of validator's activity status + """ + # Order not preserved - https://github.com/yaml/pyyaml/issues/110 + metadata = { + 'title': 'Shuffling Algorithm Tests 1', + 'summary': 'Test vectors for validator shuffling with different validator\'s activity status.' + ' Note: only relevant validator fields are defined.', + 'test_suite': 'shuffle', + 'fork': 'phase0-0.5.0', + } + + # Config + random.seed(int("0xEF00BEAC", 16)) + num_cases = 10 + + test_cases = [] + + for case in range(num_cases): + seedhash = bytes(random.randint(0, 255) for byte in range(32)) + idx_max = random.randint(128, 512) + + validators = [] + for idx in range(idx_max): + v = Validator(original_index=idx) + # 4/5 of all validators are active + if random.random() < 0.8: + # Choose a normally distributed epoch number + rand_epoch = round(random.gauss(EPOCH, RAND_EPOCH_STD)) + + # for 1/2 of *active* validators rand_epoch is the activation epoch + if random.random() < 0.5: + v.activation_epoch = rand_epoch + + # 1/4 of active validators will exit in forseeable future + if random.random() < 0.5: + v.exit_epoch = random.randint( + rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH) + # 1/4 of active validators in theory remain in the set indefinitely + else: + v.exit_epoch = FAR_FUTURE_EPOCH + # for the other active 1/2 rand_epoch is the exit epoch + else: + v.activation_epoch = random.randint( + 0, rand_epoch - ACTIVATION_EXIT_DELAY) + v.exit_epoch = rand_epoch + + # The remaining 1/5 of all validators is not activated + else: + v.activation_epoch = FAR_FUTURE_EPOCH + v.exit_epoch = FAR_FUTURE_EPOCH + + validators.append(v) + + input_ = { + 'validators': validators, + 'epoch': EPOCH + } + output = get_shuffling( + seedhash, validators, input_['epoch']) + + test_cases.append({ + 'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output + }) + + return { + 'metadata': metadata, + 'filename': 'test_vector_shuffling.yml', + 'test_cases': test_cases + } + + +def validators_set_size_variety_generator(): + """ + Different validator set size cases, inspired by removed manual `permutated_index` tests + https://github.com/ethereum/eth2.0-test-generators/tree/bcd9ab2933d9f696901d1dfda0828061e9d3093f/permutated_index + """ + # Order not preserved - https://github.com/yaml/pyyaml/issues/110 + metadata = { + 'title': 'Shuffling Algorithm Tests 2', + 'summary': 'Test vectors for validator shuffling with different validator\'s set size.' + ' Note: only relevant validator fields are defined.', + 'test_suite': 'shuffle', + 'fork': 'tchaikovsky', + 'version': 1.0 + } + + # Config + random.seed(int("0xEF00BEAC", 16)) + + test_cases = [] + + seedhash = bytes(random.randint(0, 255) for byte in range(32)) + idx_max = 4096 + set_sizes = [1, 2, 3, 1024, idx_max] + + for size in set_sizes: + validators = [] + for idx in range(size): + v = Validator(original_index=idx) + v.activation_epoch = EPOCH + v.exit_epoch = FAR_FUTURE_EPOCH + validators.append(v) + input_ = { + 'validators': validators, + 'epoch': EPOCH + } + output = get_shuffling( + seedhash, validators, input_['epoch']) + + test_cases.append({ + 'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output + }) + + return { + 'metadata': metadata, + 'filename': 'shuffling_set_size.yml', + 'test_cases': test_cases + } + + +if __name__ == '__main__': + output_dir = sys.argv[2] + for generator in [active_exited_validators_generator, validators_set_size_variety_generator]: + result = generator() + filename = os.path.join(output_dir, result['filename']) + with open(filename, 'w') as outfile: + # Dump at top level + yaml.dump(result['metadata'], outfile, default_flow_style=False) + # default_flow_style will unravel "ValidatorRecord" and "committee" line, exploding file size + yaml.dump({'test_cases': result['test_cases']}, outfile) diff --git a/test_generators/shuffling/requirements.txt b/test_generators/shuffling/requirements.txt new file mode 100644 index 0000000000..dde2fb67da --- /dev/null +++ b/test_generators/shuffling/requirements.txt @@ -0,0 +1,4 @@ +eth-hash[pycryptodome]==0.2.0 +eth-typing==2.0.0 +eth-utils==1.4.1 +PyYAML==4.2b1 diff --git a/test_generators/shuffling/utils.py b/test_generators/shuffling/utils.py new file mode 100644 index 0000000000..bcd2c6a3c1 --- /dev/null +++ b/test_generators/shuffling/utils.py @@ -0,0 +1,6 @@ +from eth_typing import Hash32 +from eth_utils import keccak + + +def hash(x: bytes) -> Hash32: + return keccak(x) diff --git a/test_generators/shuffling/yaml_objects.py b/test_generators/shuffling/yaml_objects.py new file mode 100644 index 0000000000..18e45220e6 --- /dev/null +++ b/test_generators/shuffling/yaml_objects.py @@ -0,0 +1,25 @@ +from typing import Any + +import yaml + + +class Validator(yaml.YAMLObject): + """ + A validator stub containing only the fields relevant for get_shuffling() + """ + fields = { + 'activation_epoch': 'uint64', + 'exit_epoch': 'uint64', + # Extra index field to ease testing/debugging + 'original_index': 'uint64', + } + + def __init__(self, **kwargs): + for k in self.fields.keys(): + setattr(self, k, kwargs.get(k)) + + def __setattr__(self, name: str, value: Any) -> None: + super().__setattr__(name, value) + + def __getattribute__(self, name: str) -> Any: + return super().__getattribute__(name) diff --git a/test_generators/ssz/__init__.py b/test_generators/ssz/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py new file mode 100644 index 0000000000..d19ec12b48 --- /dev/null +++ b/test_generators/ssz/main.py @@ -0,0 +1,84 @@ +import argparse +import pathlib +import sys + +from ruamel.yaml import ( + YAML, +) + +from uint_test_generators import ( + generate_uint_bounds_test, + generate_uint_random_test, + generate_uint_wrong_length_test, +) + +test_generators = [ + generate_uint_random_test, + generate_uint_wrong_length_test, + generate_uint_bounds_test, +] + + +def make_filename_for_test(test): + title = test["title"] + filename = title.lower().replace(" ", "_") + ".yaml" + return pathlib.Path(filename) + + +def validate_output_dir(path_str): + path = pathlib.Path(path_str) + + if not path.exists(): + raise argparse.ArgumentTypeError("Output directory must exist") + + if not path.is_dir(): + raise argparse.ArgumentTypeError("Output path must lead to a directory") + + return path + + +parser = argparse.ArgumentParser( + prog="gen-ssz-tests", + description="Generate YAML test files for SSZ and tree hashing", +) +parser.add_argument( + "-o", + "--output-dir", + dest="output_dir", + required=True, + type=validate_output_dir, + help="directory into which the generated YAML files will be dumped" +) +parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="if set overwrite test files if they exist", +) + + +if __name__ == "__main__": + args = parser.parse_args() + output_dir = args.output_dir + if not args.force: + file_mode = "x" + else: + file_mode = "w" + + yaml = YAML(pure=True) + + print(f"generating {len(test_generators)} test files...") + for test_generator in test_generators: + test = test_generator() + + filename = make_filename_for_test(test) + path = output_dir / filename + + try: + with path.open(file_mode) as f: + yaml.dump(test, f) + except IOError as e: + sys.exit(f'Error when dumping test "{test["title"]}" ({e})') + + print("done.") diff --git a/test_generators/ssz/renderers.py b/test_generators/ssz/renderers.py new file mode 100644 index 0000000000..e551ab14cf --- /dev/null +++ b/test_generators/ssz/renderers.py @@ -0,0 +1,102 @@ +from collections.abc import ( + Mapping, + Sequence, +) + +from eth_utils import ( + encode_hex, + to_dict, +) + +from ssz.sedes import ( + BaseSedes, + Boolean, + Bytes, + BytesN, + Container, + List, + UInt, +) + + +def render_value(value): + if isinstance(value, bool): + return value + elif isinstance(value, int): + return str(value) + elif isinstance(value, bytes): + return encode_hex(value) + elif isinstance(value, Sequence): + return tuple(render_value(element) for element in value) + elif isinstance(value, Mapping): + return render_dict_value(value) + else: + raise ValueError(f"Cannot render value {value}") + + +@to_dict +def render_dict_value(value): + for key, value in value.items(): + yield key, render_value(value) + + +def render_type_definition(sedes): + if isinstance(sedes, Boolean): + return "bool" + + elif isinstance(sedes, UInt): + return f"uint{sedes.length * 8}" + + elif isinstance(sedes, BytesN): + return f"bytes{sedes.length}" + + elif isinstance(sedes, Bytes): + return f"bytes" + + elif isinstance(sedes, List): + return [render_type_definition(sedes.element_sedes)] + + elif isinstance(sedes, Container): + return { + field_name: render_type_definition(field_sedes) + for field_name, field_sedes in sedes.fields + } + + elif isinstance(sedes, BaseSedes): + raise Exception("Unreachable: All sedes types have been checked") + + else: + raise TypeError("Expected BaseSedes") + + +@to_dict +def render_test_case(*, sedes, valid, value=None, serial=None, description=None, tags=None): + value_and_serial_given = value is not None and serial is not None + if valid: + if not value_and_serial_given: + raise ValueError("For valid test cases, both value and ssz must be present") + else: + if value_and_serial_given: + raise ValueError("For invalid test cases, either value or ssz must not be present") + + if tags is None: + tags = [] + + yield "type", render_type_definition(sedes) + yield "valid", valid + if value is not None: + yield "value", render_value(value) + if serial is not None: + yield "ssz", encode_hex(serial) + if description is not None: + yield description + yield "tags", tags + + +@to_dict +def render_test(*, title, summary, fork, test_cases): + yield "title", title, + if summary is not None: + yield "summary", summary + yield "fork", fork + yield "test_cases", test_cases diff --git a/test_generators/ssz/requirements.txt b/test_generators/ssz/requirements.txt new file mode 100644 index 0000000000..88193a01d6 --- /dev/null +++ b/test_generators/ssz/requirements.txt @@ -0,0 +1,2 @@ +ruamel.yaml==0.15.87 +ssz==0.1.0a2 diff --git a/test_generators/ssz/uint_test_generators.py b/test_generators/ssz/uint_test_generators.py new file mode 100644 index 0000000000..c8c841fe72 --- /dev/null +++ b/test_generators/ssz/uint_test_generators.py @@ -0,0 +1,132 @@ +import random + +from eth_utils import ( + to_tuple, +) + +import ssz +from ssz.sedes import ( + UInt, +) +from renderers import ( + render_test, + render_test_case, +) + +random.seed(0) + + +BIT_SIZES = [i for i in range(8, 512 + 1, 8)] +RANDOM_TEST_CASES_PER_BIT_SIZE = 10 +RANDOM_TEST_CASES_PER_LENGTH = 3 + + +def get_random_bytes(length): + return bytes(random.randint(0, 255) for _ in range(length)) + + +def generate_uint_bounds_test(): + test_cases = generate_uint_bounds_test_cases() + generate_uint_out_of_bounds_test_cases() + + return render_test( + title="UInt Bounds", + summary="Integers right at or beyond the bounds of the allowed value range", + fork="phase0-0.2.0", + test_cases=test_cases, + ) + + +def generate_uint_random_test(): + test_cases = generate_random_uint_test_cases() + + return render_test( + title="UInt Random", + summary="Random integers chosen uniformly over the allowed value range", + fork="phase0-0.2.0", + test_cases=test_cases, + ) + + +def generate_uint_wrong_length_test(): + test_cases = generate_uint_wrong_length_test_cases() + + return render_test( + title="UInt Wrong Length", + summary="Serialized integers that are too short or too long", + fork="phase0-0.2.0", + test_cases=test_cases, + ) + + +@to_tuple +def generate_random_uint_test_cases(): + for bit_size in BIT_SIZES: + sedes = UInt(bit_size) + + for _ in range(RANDOM_TEST_CASES_PER_BIT_SIZE): + value = random.randrange(0, 2 ** bit_size) + serial = ssz.encode(value, sedes) + # note that we need to create the tags in each loop cycle, otherwise ruamel will use + # YAML references which makes the resulting file harder to read + tags = tuple(["atomic", "uint", "random"]) + yield render_test_case( + sedes=sedes, + valid=True, + value=value, + serial=serial, + tags=tags, + ) + + +@to_tuple +def generate_uint_wrong_length_test_cases(): + for bit_size in BIT_SIZES: + sedes = UInt(bit_size) + lengths = sorted({ + 0, + sedes.length // 2, + sedes.length - 1, + sedes.length + 1, + sedes.length * 2, + }) + for length in lengths: + for _ in range(RANDOM_TEST_CASES_PER_LENGTH): + tags = tuple(["atomic", "uint", "wrong_length"]) + yield render_test_case( + sedes=sedes, + valid=False, + serial=get_random_bytes(length), + tags=tags, + ) + + +@to_tuple +def generate_uint_bounds_test_cases(): + common_tags = ("atomic", "uint") + for bit_size in BIT_SIZES: + sedes = UInt(bit_size) + + for value, tag in ((0, "uint_lower_bound"), (2 ** bit_size - 1, "uint_upper_bound")): + serial = ssz.encode(value, sedes) + yield render_test_case( + sedes=sedes, + valid=True, + value=value, + serial=serial, + tags=common_tags + (tag,), + ) + + +@to_tuple +def generate_uint_out_of_bounds_test_cases(): + common_tags = ("atomic", "uint") + for bit_size in BIT_SIZES: + sedes = UInt(bit_size) + + for value, tag in ((-1, "uint_underflow"), (2 ** bit_size, "uint_overflow")): + yield render_test_case( + sedes=sedes, + valid=False, + value=value, + tags=common_tags + (tag,), + ) From c7da23e6dadb3c1e7cc75d97d1dc0e7237198ecc Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 00:35:46 +0800 Subject: [PATCH 03/61] update ci config and makefile --- .circleci/config.yml | 93 +++++++++++++++++++++++++++++++++----------- Makefile | 69 ++++++++++++++++++++++++++------ 2 files changed, 127 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02871530e3..411eb92308 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,4 @@ -# Python CircleCI 2.0 configuration file -version: 2 +version: 2.1 jobs: build: docker: @@ -8,34 +7,82 @@ jobs: steps: - checkout - # Download and cache dependencies - - restore_cache: - keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - - run: - name: install dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip install -r requirements.txt - - run: - name: build phase0 spec - command: make build/phase0 - - - save_cache: - paths: - - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} + name: Build phase0 spec + command: make phase0 - run: - name: run tests + name: run py-tests command: | . venv/bin/activate pytest tests + - run: + name: Generate YAML tests + command: make yaml_tests + - store_artifacts: path: test-reports destination: test-reports + + - run: + name: Save YAML tests for deployment + command: | + mkdir /tmp/workspace + cp -r yaml_tests /tmp/workspace/ + git log -1 >> /tmp/workspace/latest_commit_message + - persist_to_workspace: + root: /tmp/workspace + paths: + - yaml_tests + - latest_commit_message + commit: + docker: + - image: circleci/python:3.6 + steps: + - attach_workspace: + at: /tmp/workspace + - add_ssh_keys: + fingerprints: + - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa" + - run: + name: Checkout test repository + command: | + ssh-keyscan -H github.com >> ~/.ssh/known_hosts + git clone git@github.com:ethereum/eth2.0-tests.git + - run: + name: Commit and push generated YAML tests + command: | + cd eth2.0-tests + git config user.name 'eth2TestGenBot' + git config user.email '47188154+eth2TestGenBot@users.noreply.github.com' + for filename in /tmp/workspace/yaml_tests/*; do + rm -rf $(basename $filename) + cp -r $filename . + done + git add . + if git diff --cached --exit-code >& /dev/null; then + echo "No changes to commit" + else + echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message + cat /tmp/workspace/latest_commit_message >> commit_message + git commit -F commit_message + git push origin master + fi +workflows: + version: 2.1 + + build_and_commit: + jobs: + - build: + filters: + tags: + only: /.*/ + - commit: + requires: + - build + filters: + tags: + only: /.*/ + branches: + ignore: /.*/ \ No newline at end of file diff --git a/Makefile b/Makefile index 88f17dcf99..f6313f64d6 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,74 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts -BUILD_DIR = ./build -UTILS_DIR = ./utils +TEST_LIBS_DIR = ./test_libs +PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec +YAML_TEST_DIR = ./yaml_tests +GENERATOR_DIR = ./test_generators +GENERATOR_VENVS_DIR = $(GENERATOR_DIR)/.venvs -.PHONY: clean all test +# Collect a list of generator names +GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) +# Map this list of generator paths to a list of test output paths +YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS)) +PY_SPEC_PHASE_0_TARGET = $(PY_SPEC_DIR)/phase0/spec.py +PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGET) -all: $(BUILD_DIR)/phase0 +.PHONY: clean all test yaml_tests pyspec phase0 + +all: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) $(PY_SPEC_ALL_TARGETS) clean: - rm -rf $(BUILD_DIR) + rm -rf $(YAML_TEST_DIR) + rm -rf $(GENERATOR_VENVS_DIR) + rm -rf $(PY_SPEC_ALL_TARGETS) +# "make yaml_tests" to run generators +yaml_tests: $(YAML_TEST_TARGETS) # runs a limited set of tests against a minimal config # run pytest with `-m` option to full suite -test: +test: $(PY_SPEC_TARGETS) pytest -m minimal_config tests/ +# "make pyspec" to create the pyspec for all phases. +pyspec: $(PY_SPEC_TARGETS) -$(BUILD_DIR)/phase0: - mkdir -p $@ +# "make phase0" to create pyspec for phase0 +phase0: $(PY_SPEC_DIR)/phase0/spec.py + + +$(PY_SPEC_DIR)/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py - mkdir -p $@/utils - cp $(UTILS_DIR)/phase0/* $@/utils - cp $(UTILS_DIR)/phase0/state_transition.py $@ - touch $@/__init__.py $@/utils/__init__.py + + + +# The function that builds a set of suite files, by calling a generator for the given type (param 1) +define build_yaml_tests + $(info running generator $(1)) + # Create the output + mkdir -p $(YAML_TEST_DIR)$(1) + + # Create a virtual environment + python3 -m venv $(VENV_DIR)$(1) + # Activate the venv, this is where dependencies are installed for the generator + . $(GENERATOR_VENVS_DIR)$(1)bin/activate + # Install all the necessary requirements + pip3 install -r $(GENERATOR_DIR)$(1)requirements.txt --user + + # Run the generator. The generator is assumed to have an "main.py" file. + # We output to the tests dir (generator program should accept a "-p " argument. + python3 $(GENERATOR_DIR)$(1)main.py -o $(YAML_TEST_DIR)$(1) + $(info generator $(1) finished) +endef + +# The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary) +$(YAML_TEST_DIR): + $(info ${YAML_TEST_TARGETS}) + mkdir -p $@ + +# For any target within the tests dir, build it using the build_yaml_tests function. +$(YAML_TEST_DIR)%: + $(call build_yaml_tests,$*) From 645682553d5e9320b4a82062f531ae79613e3bda Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 00:36:15 +0800 Subject: [PATCH 04/61] Update readme with links to spec contributor docs --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5c88daf9e..294dd439d6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https:/ This repo hosts the current eth2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed upon changes to spec can be made through pull requests. -# Specs + +## Specs Core specifications for eth2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: * [Phase 0 -- The Beacon Chain](specs/core/0_beacon-chain.md) @@ -18,10 +19,20 @@ Accompanying documents can be found in [specs](specs) and include * [General test format](specs/test-format.md) * [Honest validator implementation doc](specs/validator/0_beacon-chain-validator.md) -## Design goals + +### Design goals + The following are the broad design goals for Ethereum 2.0: * to minimize complexity, even at the cost of some losses in efficiency * to remain live through major network partitions and when very large portions of nodes go offline * to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available * to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time * to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain) + + +## For spec contributors + +Documentation on the different components used during spec writing can be found here: +* [YAML Test Generators](test_generators/README.md) +* [Executable Python Spec](test_libs/pyspec/README.md) + From bf951688b08126d526eac45234da30f49f21b1bd Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 00:43:56 +0800 Subject: [PATCH 05/61] update pyspec readme --- test_libs/pyspec/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 2747ad8f31..25ee737f79 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -1,3 +1,17 @@ # ETH 2.0 PySpec -The py \ No newline at end of file +The Python executable spec is built from the ETH 2.0 specification, + complemented with the necessary helper functions for hashing, BLS, and more. + +With this executable spec, + test-generators can easily create test-vectors for client implementations, + and the spec itself can be verified to be consistent and coherent, through sanity tests implemented with pytest. + +Contributions are welcome, but consider implementing your idea as part of the spec itself first. +The pyspec is not a replacement. +If you see opportunity to include any of the `utils/` code in the spec, + please submit an issue or PR. + +## License + +Same as the spec itself, see LICENSE file in spec repository root. From 883ea93545f8ca061523b966c48683ef049eae20 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 00:53:20 +0800 Subject: [PATCH 06/61] update generator helper readme --- test_libs/gen_helpers/README.md | 5 +++++ test_libs/gen_helpers/README.txt | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 test_libs/gen_helpers/README.md delete mode 100644 test_libs/gen_helpers/README.txt diff --git a/test_libs/gen_helpers/README.md b/test_libs/gen_helpers/README.md new file mode 100644 index 0000000000..4dcfacef73 --- /dev/null +++ b/test_libs/gen_helpers/README.md @@ -0,0 +1,5 @@ +# ETH 2.0 test generator helpers + +`gen_base`: A util to quickly write new test suite generators with. +See [Generators documentation](../../test_generators/README.md). + diff --git a/test_libs/gen_helpers/README.txt b/test_libs/gen_helpers/README.txt deleted file mode 100644 index 10c97f2d69..0000000000 --- a/test_libs/gen_helpers/README.txt +++ /dev/null @@ -1,4 +0,0 @@ -ETH 2.0 test generator helpers - -`eth2_test_gen_base`: A util to quickly write new test suite generators with. - From a106edacadf8a1724ae039c73ab76775d049c2b1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 01:21:07 +0800 Subject: [PATCH 07/61] Scope pyspec packages, make pyspec more readable, fix imports --- Makefile | 10 +- scripts/phase0/build_spec.py | 27 +- scripts/phase0/function_puller.py | 4 + test_libs/gen_helpers/setup.py | 5 +- test_libs/pyspec/{debug => eth2}/__init__.py | 0 .../pyspec/{phase0 => eth2/debug}/__init__.py | 0 test_libs/pyspec/{ => eth2}/debug/jsonize.py | 2 +- .../pyspec/{utils => eth2/phase0}/__init__.py | 0 test_libs/pyspec/eth2/phase0/spec.py | 1692 +++++++++++++++++ .../{ => eth2}/phase0/state_transition.py | 6 +- test_libs/pyspec/eth2/utils/__init__.py | 0 test_libs/pyspec/{ => eth2}/utils/bls_stub.py | 0 .../pyspec/{ => eth2}/utils/hash_function.py | 0 .../pyspec/{ => eth2}/utils/merkle_minimal.py | 0 .../pyspec/{ => eth2}/utils/minimal_ssz.py | 0 test_libs/pyspec/setup.py | 2 +- 16 files changed, 1728 insertions(+), 20 deletions(-) rename test_libs/pyspec/{debug => eth2}/__init__.py (100%) rename test_libs/pyspec/{phase0 => eth2/debug}/__init__.py (100%) rename test_libs/pyspec/{ => eth2}/debug/jsonize.py (97%) rename test_libs/pyspec/{utils => eth2/phase0}/__init__.py (100%) create mode 100644 test_libs/pyspec/eth2/phase0/spec.py rename test_libs/pyspec/{ => eth2}/phase0/state_transition.py (97%) create mode 100644 test_libs/pyspec/eth2/utils/__init__.py rename test_libs/pyspec/{ => eth2}/utils/bls_stub.py (100%) rename test_libs/pyspec/{ => eth2}/utils/hash_function.py (100%) rename test_libs/pyspec/{ => eth2}/utils/merkle_minimal.py (100%) rename test_libs/pyspec/{ => eth2}/utils/minimal_ssz.py (100%) diff --git a/Makefile b/Makefile index f6313f64d6..3812a8255d 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) # Map this list of generator paths to a list of test output paths YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS)) -PY_SPEC_PHASE_0_TARGET = $(PY_SPEC_DIR)/phase0/spec.py -PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGET) +PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2/phase0/spec.py +PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) .PHONY: clean all test yaml_tests pyspec phase0 @@ -37,11 +37,11 @@ test: $(PY_SPEC_TARGETS) pyspec: $(PY_SPEC_TARGETS) # "make phase0" to create pyspec for phase0 -phase0: $(PY_SPEC_DIR)/phase0/spec.py +phase0: $(PY_SPEC_PHASE_0_TARGETS) -$(PY_SPEC_DIR)/phase0/spec.py: - python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@/spec.py +$(PY_SPEC_DIR)/eth2/phase0/spec.py: + python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 6116f1ffe5..9f33c58834 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -4,15 +4,8 @@ def build_spec(sourcefile, outfile): code_lines = [] - - code_lines.append("from build.phase0.utils.minimal_ssz import *") - code_lines.append("from build.phase0.utils.bls_stub import *") - for i in (1, 2, 3, 4, 8, 32, 48, 96): - code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) - code_lines.append("SLOTS_PER_EPOCH = 64") # stub, will get overwritten by real var - code_lines.append("def slot_to_epoch(x): return x // SLOTS_PER_EPOCH") - code_lines.append(""" + from typing import ( Any, Callable, @@ -20,6 +13,20 @@ def build_spec(sourcefile, outfile): NewType, Tuple, ) +from eth2.utils.minimal_ssz import * +from eth2.utils.bls_stub import * + + + """) + for i in (1, 2, 3, 4, 8, 32, 48, 96): + code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) + + code_lines.append(""" +# stub, will get overwritten by real var +SLOTS_PER_EPOCH = 64 + + +def slot_to_epoch(x): return x // SLOTS_PER_EPOCH Slot = NewType('Slot', int) # uint64 @@ -40,6 +47,8 @@ def build_spec(sourcefile, outfile): # Monkey patch validator get committee code _compute_committee = compute_committee committee_cache = {} + + def compute_committee(validator_indices: List[ValidatorIndex], seed: Bytes32, index: int, @@ -60,6 +69,8 @@ def compute_committee(validator_indices: List[ValidatorIndex], # Monkey patch hash cache _hash = hash hash_cache = {} + + def hash(x): if x in hash_cache: return hash_cache[x] diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 7d5796fc7f..2cd0139c56 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -27,6 +27,10 @@ def get_lines(file_name): code_lines.append('%s = SSZType({' % current_name) processing_typedef = True elif pulling_from is not None: + # Add some whitespace between functions + if line[:3] == 'def': + code_lines.append("") + code_lines.append("") code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index a6c65c2124..88b971bf33 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -4,5 +4,8 @@ name='gen_helpers', version='1.0', packages=['gen_base'], - install_requires=['ruamel.yaml', 'eth-utils'] + install_requires=[ + "ruamel.yaml==0.15.87", + "eth-utils==1.4.1" + ] ) diff --git a/test_libs/pyspec/debug/__init__.py b/test_libs/pyspec/eth2/__init__.py similarity index 100% rename from test_libs/pyspec/debug/__init__.py rename to test_libs/pyspec/eth2/__init__.py diff --git a/test_libs/pyspec/phase0/__init__.py b/test_libs/pyspec/eth2/debug/__init__.py similarity index 100% rename from test_libs/pyspec/phase0/__init__.py rename to test_libs/pyspec/eth2/debug/__init__.py diff --git a/test_libs/pyspec/debug/jsonize.py b/test_libs/pyspec/eth2/debug/jsonize.py similarity index 97% rename from test_libs/pyspec/debug/jsonize.py rename to test_libs/pyspec/eth2/debug/jsonize.py index ac0243a5d5..660e6b070d 100644 --- a/test_libs/pyspec/debug/jsonize.py +++ b/test_libs/pyspec/eth2/debug/jsonize.py @@ -1,4 +1,4 @@ -from shared_eth2.minimal_ssz import hash_tree_root +from eth2.utils.minimal_ssz import hash_tree_root def jsonize(value, typ, include_hash_tree_roots=False): diff --git a/test_libs/pyspec/utils/__init__.py b/test_libs/pyspec/eth2/phase0/__init__.py similarity index 100% rename from test_libs/pyspec/utils/__init__.py rename to test_libs/pyspec/eth2/phase0/__init__.py diff --git a/test_libs/pyspec/eth2/phase0/spec.py b/test_libs/pyspec/eth2/phase0/spec.py new file mode 100644 index 0000000000..32dc8e7438 --- /dev/null +++ b/test_libs/pyspec/eth2/phase0/spec.py @@ -0,0 +1,1692 @@ +from eth2.utils.minimal_ssz import * +from eth2.utils.bls_stub import * +def int_to_bytes1(x): return x.to_bytes(1, 'little') +def int_to_bytes2(x): return x.to_bytes(2, 'little') +def int_to_bytes3(x): return x.to_bytes(3, 'little') +def int_to_bytes4(x): return x.to_bytes(4, 'little') +def int_to_bytes8(x): return x.to_bytes(8, 'little') +def int_to_bytes32(x): return x.to_bytes(32, 'little') +def int_to_bytes48(x): return x.to_bytes(48, 'little') +def int_to_bytes96(x): return x.to_bytes(96, 'little') + + +SLOTS_PER_EPOCH = 64 +def slot_to_epoch(x): return x // SLOTS_PER_EPOCH + + +from typing import ( + Any, + Callable, + List, + NewType, + Tuple, +) + + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Any = None +Store = None + +SHARD_COUNT = 2**10 +TARGET_COMMITTEE_SIZE = 2**7 +MAX_BALANCE_CHURN_QUOTIENT = 2**5 +MAX_SLASHABLE_ATTESTATION_PARTICIPANTS = 2**12 +MAX_EXIT_DEQUEUES_PER_EPOCH = 2**2 +SHUFFLE_ROUND_COUNT = 90 +DEPOSIT_CONTRACT_ADDRESS = 0x1234567890123567890123456789012357890 +DEPOSIT_CONTRACT_TREE_DEPTH = 2**5 +MIN_DEPOSIT_AMOUNT = 2**0 * 10**9 +MAX_DEPOSIT_AMOUNT = 2**5 * 10**9 +EJECTION_BALANCE = 2**4 * 10**9 +HIGH_BALANCE_INCREMENT = 2**0 * 10**9 +GENESIS_FORK_VERSION = int_to_bytes4(0) +GENESIS_SLOT = 2**32 +GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) +GENESIS_START_SHARD = 0 +FAR_FUTURE_EPOCH = 2**64 - 1 +ZERO_HASH = int_to_bytes32(0) +EMPTY_SIGNATURE = int_to_bytes96(0) +BLS_WITHDRAWAL_PREFIX_BYTE = int_to_bytes1(0) +SECONDS_PER_SLOT = 6 +MIN_ATTESTATION_INCLUSION_DELAY = 2**2 +SLOTS_PER_EPOCH = 2**6 +MIN_SEED_LOOKAHEAD = 2**0 +ACTIVATION_EXIT_DELAY = 2**2 +EPOCHS_PER_ETH1_VOTING_PERIOD = 2**4 +SLOTS_PER_HISTORICAL_ROOT = 2**13 +MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8 +PERSISTENT_COMMITTEE_PERIOD = 2**11 +MAX_CROSSLINK_EPOCHS = 2**6 +LATEST_RANDAO_MIXES_LENGTH = 2**13 +LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 2**13 +LATEST_SLASHED_EXIT_LENGTH = 2**13 +BASE_REWARD_QUOTIENT = 2**5 +WHISTLEBLOWER_REWARD_QUOTIENT = 2**9 +ATTESTATION_INCLUSION_REWARD_QUOTIENT = 2**3 +INACTIVITY_PENALTY_QUOTIENT = 2**24 +MIN_PENALTY_QUOTIENT = 2**5 +MAX_PROPOSER_SLASHINGS = 2**4 +MAX_ATTESTER_SLASHINGS = 2**0 +MAX_ATTESTATIONS = 2**7 +MAX_DEPOSITS = 2**4 +MAX_VOLUNTARY_EXITS = 2**4 +MAX_TRANSFERS = 2**4 +DOMAIN_BEACON_BLOCK = 0 +DOMAIN_RANDAO = 1 +DOMAIN_ATTESTATION = 2 +DOMAIN_DEPOSIT = 3 +DOMAIN_VOLUNTARY_EXIT = 4 +DOMAIN_TRANSFER = 5 +Fork = SSZType({ + # Previous fork version + 'previous_version': 'bytes4', + # Current fork version + 'current_version': 'bytes4', + # Fork epoch number + 'epoch': 'uint64', +}) +Crosslink = SSZType({ + # Epoch number + 'epoch': 'uint64', + # Shard data since the previous crosslink + 'crosslink_data_root': 'bytes32', +}) +Eth1Data = SSZType({ + # Root of the deposit tree + 'deposit_root': 'bytes32', + # Total number of deposits + 'deposit_count': 'uint64', + # Block hash + 'block_hash': 'bytes32', +}) +Eth1DataVote = SSZType({ + # Data being voted for + 'eth1_data': Eth1Data, + # Vote count + 'vote_count': 'uint64', +}) +AttestationData = SSZType({ + # LMD GHOST vote + 'slot': 'uint64', + 'beacon_block_root': 'bytes32', + + # FFG vote + 'source_epoch': 'uint64', + 'source_root': 'bytes32', + 'target_root': 'bytes32', + + # Crosslink vote + 'shard': 'uint64', + 'previous_crosslink': Crosslink, + 'crosslink_data_root': 'bytes32', +}) +AttestationDataAndCustodyBit = SSZType({ + # Attestation data + 'data': AttestationData, + # Custody bit + 'custody_bit': 'bool', +}) +SlashableAttestation = SSZType({ + # Validator indices + 'validator_indices': ['uint64'], + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # Aggregate signature + 'aggregate_signature': 'bytes96', +}) +DepositData = SSZType({ + # BLS pubkey + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # Amount in Gwei + 'amount': 'uint64', + # Container self-signature + 'proof_of_possession': 'bytes96', +}) +BeaconBlockHeader = SSZType({ + 'slot': 'uint64', + 'previous_block_root': 'bytes32', + 'state_root': 'bytes32', + 'block_body_root': 'bytes32', + 'signature': 'bytes96', +}) +Validator = SSZType({ + # BLS public key + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # Epoch when validator activated + 'activation_epoch': 'uint64', + # Epoch when validator exited + 'exit_epoch': 'uint64', + # Epoch when validator is eligible to withdraw + 'withdrawable_epoch': 'uint64', + # Did the validator initiate an exit + 'initiated_exit': 'bool', + # Was the validator slashed + 'slashed': 'bool', + # Rounded balance + 'high_balance': 'uint64' +}) +PendingAttestation = SSZType({ + # Attester aggregation bitfield + 'aggregation_bitfield': 'bytes', + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # Inclusion slot + 'inclusion_slot': 'uint64', +}) +HistoricalBatch = SSZType({ + # Block roots + 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + # State roots + 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], +}) +ProposerSlashing = SSZType({ + # Proposer index + 'proposer_index': 'uint64', + # First block header + 'header_1': BeaconBlockHeader, + # Second block header + 'header_2': BeaconBlockHeader, +}) +AttesterSlashing = SSZType({ + # First slashable attestation + 'slashable_attestation_1': SlashableAttestation, + # Second slashable attestation + 'slashable_attestation_2': SlashableAttestation, +}) +Attestation = SSZType({ + # Attester aggregation bitfield + 'aggregation_bitfield': 'bytes', + # Attestation data + 'data': AttestationData, + # Custody bitfield + 'custody_bitfield': 'bytes', + # BLS aggregate signature + 'aggregate_signature': 'bytes96', +}) +Deposit = SSZType({ + # Branch in the deposit tree + 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], + # Index in the deposit tree + 'index': 'uint64', + # Data + 'data': DepositData, +}) +VoluntaryExit = SSZType({ + # Minimum epoch for processing exit + 'epoch': 'uint64', + # Index of the exiting validator + 'validator_index': 'uint64', + # Validator signature + 'signature': 'bytes96', +}) +Transfer = SSZType({ + # Sender index + 'sender': 'uint64', + # Recipient index + 'recipient': 'uint64', + # Amount in Gwei + 'amount': 'uint64', + # Fee in Gwei for block proposer + 'fee': 'uint64', + # Inclusion slot + 'slot': 'uint64', + # Sender withdrawal pubkey + 'pubkey': 'bytes48', + # Sender signature + 'signature': 'bytes96', +}) +BeaconBlockBody = SSZType({ + 'randao_reveal': 'bytes96', + 'eth1_data': Eth1Data, + 'proposer_slashings': [ProposerSlashing], + 'attester_slashings': [AttesterSlashing], + 'attestations': [Attestation], + 'deposits': [Deposit], + 'voluntary_exits': [VoluntaryExit], + 'transfers': [Transfer], +}) +BeaconBlock = SSZType({ + # Header + 'slot': 'uint64', + 'previous_block_root': 'bytes32', + 'state_root': 'bytes32', + 'body': BeaconBlockBody, + 'signature': 'bytes96', +}) +BeaconState = SSZType({ + # Misc + 'slot': 'uint64', + 'genesis_time': 'uint64', + 'fork': Fork, # For versioning hard forks + + # Validator registry + 'validator_registry': [Validator], + 'balances': ['uint64'], + 'validator_registry_update_epoch': 'uint64', + + # Randomness and committees + 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], + 'latest_start_shard': 'uint64', + + # Finality + 'previous_epoch_attestations': [PendingAttestation], + 'current_epoch_attestations': [PendingAttestation], + 'previous_justified_epoch': 'uint64', + 'current_justified_epoch': 'uint64', + 'previous_justified_root': 'bytes32', + 'current_justified_root': 'bytes32', + 'justification_bitfield': 'uint64', + 'finalized_epoch': 'uint64', + 'finalized_root': 'bytes32', + + # Recent state + 'latest_crosslinks': [Crosslink, SHARD_COUNT], + 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], + 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period + 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily + 'historical_roots': ['bytes32'], + + # Ethereum 1.0 chain data + 'latest_eth1_data': Eth1Data, + 'eth1_data_votes': [Eth1DataVote], + 'deposit_index': 'uint64', +}) + + +def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: + return bytes(a ^ b for a, b in zip(bytes1, bytes2)) + + +def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: + """ + Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. + """ + return BeaconBlockHeader( + slot=block.slot, + previous_block_root=block.previous_block_root, + state_root=ZERO_HASH, + block_body_root=hash_tree_root(block.body), + # signed_root(block) is used for block id purposes so signature is a stub + signature=EMPTY_SIGNATURE, + ) + + +def slot_to_epoch(slot: Slot) -> Epoch: + """ + Return the epoch number of the given ``slot``. + """ + return slot // SLOTS_PER_EPOCH + + +def get_previous_epoch(state: BeaconState) -> Epoch: + """` + Return the previous epoch of the given ``state``. + """ + return get_current_epoch(state) - 1 + + +def get_current_epoch(state: BeaconState) -> Epoch: + """ + Return the current epoch of the given ``state``. + """ + return slot_to_epoch(state.slot) + + +def get_epoch_start_slot(epoch: Epoch) -> Slot: + """ + Return the starting slot of the given ``epoch``. + """ + return epoch * SLOTS_PER_EPOCH + + +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch + + +def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is slashable. + """ + return ( + validator.activation_epoch <= epoch < validator.withdrawable_epoch and + validator.slashed is False + ) + + +def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: + """ + Get indices of active validators from ``validators``. + """ + return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] + + +def get_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: + """ + Return the balance for a validator with the given ``index``. + """ + return state.balances[index] + + +def set_balance(state: BeaconState, index: ValidatorIndex, balance: Gwei) -> None: + """ + Set the balance for a validator with the given ``index`` in both ``BeaconState`` + and validator's rounded balance ``high_balance``. + """ + validator = state.validator_registry[index] + HALF_INCREMENT = HIGH_BALANCE_INCREMENT // 2 + if validator.high_balance > balance or validator.high_balance + 3 * HALF_INCREMENT < balance: + validator.high_balance = balance - balance % HIGH_BALANCE_INCREMENT + state.balances[index] = balance + + +def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Increase the balance for a validator with the given ``index`` by ``delta``. + """ + set_balance(state, index, get_balance(state, index) + delta) + + +def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Decrease the balance for a validator with the given ``index`` by ``delta``. + Set to ``0`` when underflow. + """ + current_balance = get_balance(state, index) + set_balance(state, index, current_balance - delta if current_balance >= delta else 0) + + +def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: + """ + Return `p(index)` in a pseudorandom permutation `p` of `0...list_size - 1` with ``seed`` as entropy. + + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. + """ + assert index < list_size + assert list_size <= 2**40 + + for round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size + flip = (pivot - index) % list_size + position = max(index, flip) + source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) + byte = source[(position % 256) // 8] + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index + + +def get_split_offset(list_size: int, chunks: int, index: int) -> int: + """ + Returns a value such that for a list L, chunk count k and index i, + split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k, i+1)] + """ + return (list_size * index) // chunks + + +def get_epoch_committee_count(active_validator_count: int) -> int: + """ + Return the number of committees in one epoch. + """ + return max( + 1, + min( + SHARD_COUNT // SLOTS_PER_EPOCH, + active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + ) + ) * SLOTS_PER_EPOCH + + +def compute_committee(validator_indices: List[ValidatorIndex], + seed: Bytes32, + index: int, + total_committees: int) -> List[ValidatorIndex]: + """ + Return the ``index``'th shuffled committee out of a total ``total_committees`` + using ``validator_indices`` and ``seed``. + """ + start_offset = get_split_offset(len(validator_indices), total_committees, index) + end_offset = get_split_offset(len(validator_indices), total_committees, index + 1) + return [ + validator_indices[get_permuted_index(i, len(validator_indices), seed)] + for i in range(start_offset, end_offset) + ] + + +def get_current_epoch_committee_count(state: BeaconState) -> int: + """ + Return the number of committees in the current epoch of the given ``state``. + """ + current_active_validators = get_active_validator_indices( + state.validator_registry, + get_current_epoch(state), + ) + return get_epoch_committee_count(len(current_active_validators)) + + +def get_crosslink_committees_at_slot(state: BeaconState, + slot: Slot) -> List[Tuple[List[ValidatorIndex], Shard]]: + """ + Return the list of ``(committee, shard)`` tuples for the ``slot``. + """ + epoch = slot_to_epoch(slot) + current_epoch = get_current_epoch(state) + previous_epoch = get_previous_epoch(state) + next_epoch = current_epoch + 1 + + assert previous_epoch <= epoch <= next_epoch + indices = get_active_validator_indices( + state.validator_registry, + epoch, + ) + committees_per_epoch = get_epoch_committee_count(len(indices)) + + if epoch == current_epoch: + start_shard = state.latest_start_shard + elif epoch == previous_epoch: + start_shard = (state.latest_start_shard - committees_per_epoch) % SHARD_COUNT + elif epoch == next_epoch: + current_epoch_committees = get_current_epoch_committee_count(state) + start_shard = (state.latest_start_shard + current_epoch_committees) % SHARD_COUNT + + committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH + offset = slot % SLOTS_PER_EPOCH + slot_start_shard = (start_shard + committees_per_slot * offset) % SHARD_COUNT + seed = generate_seed(state, epoch) + + return [ + ( + compute_committee(indices, seed, committees_per_slot * offset + i, committees_per_epoch), + (slot_start_shard + i) % SHARD_COUNT, + ) + for i in range(committees_per_slot) + ] + + +def get_block_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the block root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] + + +def get_state_root(state: BeaconState, + slot: Slot) -> Bytes32: + """ + Return the state root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] + + +def get_randao_mix(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Return the randao mix at a recent ``epoch``. + """ + assert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch <= get_current_epoch(state) + return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] + + +def get_active_index_root(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Return the index root at a recent ``epoch``. + """ + assert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY < epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY + return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + + +def generate_seed(state: BeaconState, + epoch: Epoch) -> Bytes32: + """ + Generate a seed for the given ``epoch``. + """ + return hash( + get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD) + + get_active_index_root(state, epoch) + + int_to_bytes32(epoch) + ) + + +def get_beacon_proposer_index(state: BeaconState, + slot: Slot) -> ValidatorIndex: + """ + Return the beacon proposer index for the ``slot``. + Due to proposer selection being based upon the validator balances during + the epoch in question, this can only be run for the current epoch. + """ + current_epoch = get_current_epoch(state) + assert slot_to_epoch(slot) == current_epoch + + first_committee, _ = get_crosslink_committees_at_slot(state, slot)[0] + i = 0 + while True: + rand_byte = hash( + generate_seed(state, current_epoch) + + int_to_bytes8(i // 32) + )[i % 32] + candidate = first_committee[(current_epoch + i) % len(first_committee)] + if get_effective_balance(state, candidate) * 256 > MAX_DEPOSIT_AMOUNT * rand_byte: + return candidate + i += 1 + + +def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: + """ + Verify that the given ``leaf`` is on the merkle branch ``proof`` + starting with the given ``root``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(proof[i] + value) + else: + value = hash(value + proof[i]) + return value == root + + +def get_attestation_participants(state: BeaconState, + attestation_data: AttestationData, + bitfield: bytes) -> List[ValidatorIndex]: + """ + Return the participant indices corresponding to ``attestation_data`` and ``bitfield``. + """ + # Find the committee in the list with the desired shard + crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) + + assert attestation_data.shard in [shard for _, shard in crosslink_committees] + crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] + + assert verify_bitfield(bitfield, len(crosslink_committee)) + + # Find the participating attesters in the committee + participants = [] + for i, validator_index in enumerate(crosslink_committee): + aggregation_bit = get_bitfield_bit(bitfield, i) + if aggregation_bit == 0b1: + participants.append(validator_index) + return participants + + +def bytes_to_int(data: bytes) -> int: + return int.from_bytes(data, 'little') + + +def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: + """ + Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. + """ + return min(get_balance(state, index), MAX_DEPOSIT_AMOUNT) + + +def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of an array of ``validators``. + """ + return sum([get_effective_balance(state, i) for i in validators]) + + +def get_fork_version(fork: Fork, + epoch: Epoch) -> bytes: + """ + Return the fork version of the given ``epoch``. + """ + if epoch < fork.epoch: + return fork.previous_version + else: + return fork.current_version + + +def get_domain(fork: Fork, + epoch: Epoch, + domain_type: int) -> int: + """ + Get the domain number that represents the fork meta and signature domain. + """ + return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) + + +def get_bitfield_bit(bitfield: bytes, i: int) -> int: + """ + Extract the bit in ``bitfield`` at position ``i``. + """ + return (bitfield[i // 8] >> (i % 8)) % 2 + + +def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: + """ + Verify ``bitfield`` against the ``committee_size``. + """ + if len(bitfield) != (committee_size + 7) // 8: + return False + + # Check `bitfield` is padded with zero bits only + for i in range(committee_size, len(bitfield) * 8): + if get_bitfield_bit(bitfield, i) == 0b1: + return False + + return True + + +def verify_slashable_attestation(state: BeaconState, slashable_attestation: SlashableAttestation) -> bool: + """ + Verify validity of ``slashable_attestation`` fields. + """ + if slashable_attestation.custody_bitfield != b'\x00' * len(slashable_attestation.custody_bitfield): # [TO BE REMOVED IN PHASE 1] + return False + + if not (1 <= len(slashable_attestation.validator_indices) <= MAX_SLASHABLE_ATTESTATION_PARTICIPANTS): + return False + + for i in range(len(slashable_attestation.validator_indices) - 1): + if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1]: + return False + + if not verify_bitfield(slashable_attestation.custody_bitfield, len(slashable_attestation.validator_indices)): + return False + + custody_bit_0_indices = [] + custody_bit_1_indices = [] + for i, validator_index in enumerate(slashable_attestation.validator_indices): + if get_bitfield_bit(slashable_attestation.custody_bitfield, i) == 0b0: + custody_bit_0_indices.append(validator_index) + else: + custody_bit_1_indices.append(validator_index) + + return bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), + ], + signature=slashable_attestation.aggregate_signature, + domain=get_domain(state.fork, slot_to_epoch(slashable_attestation.data.slot), DOMAIN_ATTESTATION), + ) + + +def is_double_vote(attestation_data_1: AttestationData, + attestation_data_2: AttestationData) -> bool: + """ + Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. + """ + target_epoch_1 = slot_to_epoch(attestation_data_1.slot) + target_epoch_2 = slot_to_epoch(attestation_data_2.slot) + return target_epoch_1 == target_epoch_2 + + +def is_surround_vote(attestation_data_1: AttestationData, + attestation_data_2: AttestationData) -> bool: + """ + Check if ``attestation_data_1`` surrounds ``attestation_data_2``. + """ + source_epoch_1 = attestation_data_1.source_epoch + source_epoch_2 = attestation_data_2.source_epoch + target_epoch_1 = slot_to_epoch(attestation_data_1.slot) + target_epoch_2 = slot_to_epoch(attestation_data_2.slot) + + return source_epoch_1 < source_epoch_2 and target_epoch_2 < target_epoch_1 + + +def integer_squareroot(n: int) -> int: + """ + The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. + """ + assert n >= 0 + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x + + +def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. + """ + return epoch + 1 + ACTIVATION_EXIT_DELAY + + +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + """ + Process a deposit from Ethereum 1.0. + Note that this function mutates ``state``. + """ + # Deposits must be processed in order + assert deposit.index == state.deposit_index + + # Verify the Merkle branch + merkle_branch_is_valid = verify_merkle_branch( + leaf=hash(serialize(deposit.data)), # 48 + 32 + 8 + 96 = 184 bytes serialization + proof=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH, + index=deposit.index, + root=state.latest_eth1_data.deposit_root, + ) + assert merkle_branch_is_valid + + # Increment the next deposit index we are expecting. Note that this + # needs to be done here because while the deposit contract will never + # create an invalid Merkle branch, it may admit an invalid deposit + # object, and we need to be able to skip over it + state.deposit_index += 1 + + validator_pubkeys = [v.pubkey for v in state.validator_registry] + pubkey = deposit.data.pubkey + amount = deposit.data.amount + + if pubkey not in validator_pubkeys: + # Verify the proof of possession + proof_is_valid = bls_verify( + pubkey=pubkey, + message_hash=signed_root(deposit.data), + signature=deposit.data.proof_of_possession, + domain=get_domain( + state.fork, + get_current_epoch(state), + DOMAIN_DEPOSIT, + ) + ) + if not proof_is_valid: + return + + # Add new validator + validator = Validator( + pubkey=pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + initiated_exit=False, + slashed=False, + high_balance=0 + ) + + # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. + state.validator_registry.append(validator) + state.balances.append(0) + set_balance(state, len(state.validator_registry) - 1, amount) + else: + # Increase balance by deposit amount + index = validator_pubkeys.index(pubkey) + increase_balance(state, index, amount) + + +def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: + """ + Activate the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + + validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) + + +def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: + """ + Initiate the validator of the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + validator.initiated_exit = True + + +def exit_validator(state: BeaconState, index: ValidatorIndex) -> None: + """ + Exit the validator with the given ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + + # Update validator exit epoch if not previously exited + if validator.exit_epoch == FAR_FUTURE_EPOCH: + validator.exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) + + +def slash_validator(state: BeaconState, index: ValidatorIndex) -> None: + """ + Slash the validator with index ``index``. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + exit_validator(state, index) + state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += get_effective_balance(state, index) + + whistleblower_index = get_beacon_proposer_index(state, state.slot) + whistleblower_reward = get_effective_balance(state, index) // WHISTLEBLOWER_REWARD_QUOTIENT + increase_balance(state, whistleblower_index, whistleblower_reward) + decrease_balance(state, index, whistleblower_reward) + validator.slashed = True + validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH + + +def prepare_validator_for_withdrawal(state: BeaconState, index: ValidatorIndex) -> None: + """ + Set the validator with the given ``index`` as withdrawable + ``MIN_VALIDATOR_WITHDRAWABILITY_DELAY`` after the current epoch. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[index] + validator.withdrawable_epoch = get_current_epoch(state) + MIN_VALIDATOR_WITHDRAWABILITY_DELAY + + +def get_empty_block() -> BeaconBlock: + """ + Get an empty ``BeaconBlock``. + """ + return BeaconBlock( + slot=GENESIS_SLOT, + previous_block_root=ZERO_HASH, + state_root=ZERO_HASH, + body=BeaconBlockBody( + randao_reveal=EMPTY_SIGNATURE, + eth1_data=Eth1Data( + deposit_root=ZERO_HASH, + deposit_count=0, + block_hash=ZERO_HASH, + ), + proposer_slashings=[], + attester_slashings=[], + attestations=[], + deposits=[], + voluntary_exits=[], + transfers=[], + ), + signature=EMPTY_SIGNATURE, + ) + + +def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], + genesis_time: int, + genesis_eth1_data: Eth1Data) -> BeaconState: + """ + Get the genesis ``BeaconState``. + """ + state = BeaconState( + # Misc + slot=GENESIS_SLOT, + genesis_time=genesis_time, + fork=Fork( + previous_version=GENESIS_FORK_VERSION, + current_version=GENESIS_FORK_VERSION, + epoch=GENESIS_EPOCH, + ), + + # Validator registry + validator_registry=[], + balances=[], + validator_registry_update_epoch=GENESIS_EPOCH, + + # Randomness and committees + latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), + latest_start_shard=GENESIS_START_SHARD, + + # Finality + previous_epoch_attestations=[], + current_epoch_attestations=[], + previous_justified_epoch=GENESIS_EPOCH - 1, + current_justified_epoch=GENESIS_EPOCH, + previous_justified_root=ZERO_HASH, + current_justified_root=ZERO_HASH, + justification_bitfield=0, + finalized_epoch=GENESIS_EPOCH, + finalized_root=ZERO_HASH, + + # Recent state + latest_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), + latest_block_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), + latest_state_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), + latest_active_index_roots=Vector([ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)]), + latest_slashed_balances=Vector([0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)]), + latest_block_header=get_temporary_block_header(get_empty_block()), + historical_roots=[], + + # Ethereum 1.0 chain data + latest_eth1_data=genesis_eth1_data, + eth1_data_votes=[], + deposit_index=0, + ) + + # Process genesis deposits + for deposit in genesis_validator_deposits: + process_deposit(state, deposit) + + # Process genesis activations + for validator_index, _ in enumerate(state.validator_registry): + if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: + activate_validator(state, validator_index, is_genesis=True) + + genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) + for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): + state.latest_active_index_roots[index] = genesis_active_index_root + + return state + + +def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: + """ + Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. + """ + if block.slot == slot: + return block + elif block.slot < slot: + return None + else: + return get_ancestor(store, store.get_parent(block), slot) + + +def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: + """ + Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. + """ + validators = start_state.validator_registry + active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) + attestation_targets = [ + (validator_index, get_latest_attestation_target(store, validator_index)) + for validator_index in active_validator_indices + ] + + # Use the rounded-balance-with-hysteresis supplied by the protocol for fork + # choice voting. This reduces the number of recomputations that need to be + # made for optimized implementations that precompute and save data + def get_vote_count(block: BeaconBlock) -> int: + return sum( + start_state.validator_registry[validator_index].high_balance + for validator_index, target in attestation_targets + if get_ancestor(store, target, block.slot) == block + ) + + head = start_block + while 1: + children = get_children(store, head) + if len(children) == 0: + return head + head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) + + +def cache_state(state: BeaconState) -> None: + previous_slot_state_root = hash_tree_root(state) + + # store the previous slot's post state transition root + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root + + # cache state root in stored latest_block_header if empty + if state.latest_block_header.state_root == ZERO_HASH: + state.latest_block_header.state_root = previous_slot_state_root + + # store latest known block for previous slot + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = signed_root(state.latest_block_header) + + +def get_current_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_current_epoch(state))) + + +def get_previous_total_balance(state: BeaconState) -> Gwei: + return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_previous_epoch(state))) + + +def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: + output = set() + for a in attestations: + output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) + return sorted(list(output)) + + +def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: + return get_total_balance(state, get_attesting_indices(state, attestations)) + + +def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.current_epoch_attestations + if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) + ] + + +def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.previous_epoch_attestations + if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) + ] + + +def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: + return [ + a for a in state.previous_epoch_attestations + if a.data.beacon_block_root == get_block_root(state, a.data.slot) + ] + + +def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: + all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations + valid_attestations = [ + a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard] + ] + all_roots = [a.data.crosslink_data_root for a in valid_attestations] + + # handle when no attestations for shard available + if len(all_roots) == 0: + return ZERO_HASH, [] + + def get_attestations_for(root) -> List[PendingAttestation]: + return [a for a in valid_attestations if a.data.crosslink_data_root == root] + + # Winning crosslink root is the root with the most votes for it, ties broken in favor of + # lexicographically higher hash + winning_root = max(all_roots, key=lambda r: (get_attesting_balance(state, get_attestations_for(r)), r)) + + return winning_root, get_attesting_indices(state, get_attestations_for(winning_root)) + + +def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation: + return min([ + a for a in state.previous_epoch_attestations if + validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield) + ], key=lambda a: a.inclusion_slot) + + +def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot: + return earliest_attestation(state, validator_index).inclusion_slot + + +def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int: + attestation = earliest_attestation(state, validator_index) + return attestation.inclusion_slot - attestation.data.slot + + +def update_justification_and_finalization(state: BeaconState) -> None: + new_justified_epoch = state.current_justified_epoch + new_finalized_epoch = state.finalized_epoch + + # Rotate the justification bitfield up one epoch to make room for the current epoch + state.justification_bitfield <<= 1 + # If the previous epoch gets justified, fill the second last bit + previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) + if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: + new_justified_epoch = get_current_epoch(state) - 1 + state.justification_bitfield |= 2 + # If the current epoch gets justified, fill the last bit + current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) + if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: + new_justified_epoch = get_current_epoch(state) + state.justification_bitfield |= 1 + + # Process finalizations + bitfield = state.justification_bitfield + current_epoch = get_current_epoch(state) + # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source + if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3: + new_finalized_epoch = state.previous_justified_epoch + # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source + if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2: + new_finalized_epoch = state.previous_justified_epoch + # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source + if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2: + new_finalized_epoch = state.current_justified_epoch + # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source + if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1: + new_finalized_epoch = state.current_justified_epoch + + # Update state jusification/finality fields + state.previous_justified_epoch = state.current_justified_epoch + state.previous_justified_root = state.current_justified_root + if new_justified_epoch != state.current_justified_epoch: + state.current_justified_epoch = new_justified_epoch + state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch)) + if new_finalized_epoch != state.finalized_epoch: + state.finalized_epoch = new_finalized_epoch + state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch)) + + +def process_crosslinks(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + previous_epoch = max(current_epoch - 1, GENESIS_EPOCH) + next_epoch = current_epoch + 1 + for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): + for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): + winning_root, participants = get_winning_root_and_participants(state, shard) + participating_balance = get_total_balance(state, participants) + total_balance = get_total_balance(state, crosslink_committee) + if 3 * participating_balance >= 2 * total_balance: + state.latest_crosslinks[shard] = Crosslink( + epoch=min(slot_to_epoch(slot), state.latest_crosslinks[shard].epoch + MAX_CROSSLINK_EPOCHS), + crosslink_data_root=winning_root + ) + + +def maybe_reset_eth1_period(state: BeaconState) -> None: + if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + for eth1_data_vote in state.eth1_data_votes: + # If a majority of all votes were for a particular eth1_data value, + # then set that as the new canonical value + if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: + state.latest_eth1_data = eth1_data_vote.eth1_data + state.eth1_data_votes = [] + + +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + if get_previous_total_balance(state) == 0: + return 0 + + adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT + return get_effective_balance(state, index) // adjusted_quotient // 5 + + +def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: + if epochs_since_finality <= 4: + extra_penalty = 0 + else: + extra_penalty = get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 + return get_base_reward(state, index) + extra_penalty + + +def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + current_epoch = get_current_epoch(state) + epochs_since_finality = current_epoch + 1 - state.finalized_epoch + rewards = [0 for index in range(len(state.validator_registry))] + penalties = [0 for index in range(len(state.validator_registry))] + # Some helper variables + boundary_attestations = get_previous_epoch_boundary_attestations(state) + boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) + total_balance = get_previous_total_balance(state) + total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) + matching_head_attestations = get_previous_epoch_matching_head_attestations(state) + matching_head_balance = get_attesting_balance(state, matching_head_attestations) + eligible_validators = [ + index for index, validator in enumerate(state.validator_registry) + if ( + is_active_validator(validator, current_epoch) or + (validator.slashed and current_epoch < validator.withdrawable_epoch) + ) + ] + # Process rewards or penalties for all validators + for index in eligible_validators: + base_reward = get_base_reward(state, index) + # Expected FFG source + if index in get_attesting_indices(state, state.previous_epoch_attestations): + rewards[index] += base_reward * total_attesting_balance // total_balance + # Inclusion speed bonus + rewards[index] += ( + base_reward * MIN_ATTESTATION_INCLUSION_DELAY // + inclusion_distance(state, index) + ) + else: + penalties[index] += base_reward + # Expected FFG target + if index in get_attesting_indices(state, boundary_attestations): + rewards[index] += base_reward * boundary_attesting_balance // total_balance + else: + penalties[index] += get_inactivity_penalty(state, index, epochs_since_finality) + # Expected head + if index in get_attesting_indices(state, matching_head_attestations): + rewards[index] += base_reward * matching_head_balance // total_balance + else: + penalties[index] += base_reward + # Proposer bonus + if index in get_attesting_indices(state, state.previous_epoch_attestations): + proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index)) + rewards[proposer_index] += base_reward // ATTESTATION_INCLUSION_REWARD_QUOTIENT + # Take away max rewards if we're not finalizing + if epochs_since_finality > 4: + penalties[index] += base_reward * 4 + return [rewards, penalties] + + +def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: + rewards = [0 for index in range(len(state.validator_registry))] + penalties = [0 for index in range(len(state.validator_registry))] + previous_epoch_start_slot = get_epoch_start_slot(get_previous_epoch(state)) + current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + for slot in range(previous_epoch_start_slot, current_epoch_start_slot): + for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): + winning_root, participants = get_winning_root_and_participants(state, shard) + participating_balance = get_total_balance(state, participants) + total_balance = get_total_balance(state, crosslink_committee) + for index in crosslink_committee: + if index in participants: + rewards[index] += get_base_reward(state, index) * participating_balance // total_balance + else: + penalties[index] += get_base_reward(state, index) + return [rewards, penalties] + + +def apply_rewards(state: BeaconState) -> None: + rewards1, penalties1 = get_justification_and_finalization_deltas(state) + rewards2, penalties2 = get_crosslink_deltas(state) + for i in range(len(state.validator_registry)): + set_balance( + state, + i, + max( + 0, + get_balance(state, i) + rewards1[i] + rewards2[i] - penalties1[i] - penalties2[i], + ), + ) + + +def process_ejections(state: BeaconState) -> None: + """ + Iterate through the validator registry + and eject active validators with balance below ``EJECTION_BALANCE``. + """ + for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)): + if get_balance(state, index) < EJECTION_BALANCE: + initiate_validator_exit(state, index) + + +def update_validator_registry(state: BeaconState) -> None: + """ + Update validator registry. + Note that this function mutates ``state``. + """ + current_epoch = get_current_epoch(state) + # The active validators + active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) + # The total effective balance of active validators + total_balance = get_total_balance(state, active_validator_indices) + + # The maximum balance churn in Gwei (for deposits and exits separately) + max_balance_churn = max( + MAX_DEPOSIT_AMOUNT, + total_balance // (2 * MAX_BALANCE_CHURN_QUOTIENT) + ) + + # Activate validators within the allowable balance churn + balance_churn = 0 + for index, validator in enumerate(state.validator_registry): + if validator.activation_epoch == FAR_FUTURE_EPOCH and get_balance(state, index) >= MAX_DEPOSIT_AMOUNT: + # Check the balance churn would be within the allowance + balance_churn += get_effective_balance(state, index) + if balance_churn > max_balance_churn: + break + + # Activate validator + activate_validator(state, index, is_genesis=False) + + # Exit validators within the allowable balance churn + if current_epoch < state.validator_registry_update_epoch + LATEST_SLASHED_EXIT_LENGTH: + balance_churn = ( + state.latest_slashed_balances[state.validator_registry_update_epoch % LATEST_SLASHED_EXIT_LENGTH] - + state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + ) + + for index, validator in enumerate(state.validator_registry): + if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit: + # Check the balance churn would be within the allowance + balance_churn += get_effective_balance(state, index) + if balance_churn > max_balance_churn: + break + + # Exit validator + exit_validator(state, index) + + state.validator_registry_update_epoch = current_epoch + + +def update_registry(state: BeaconState) -> None: + # Check if we should update, and if so, update + if state.finalized_epoch > state.validator_registry_update_epoch: + update_validator_registry(state) + state.latest_start_shard = ( + state.latest_start_shard + + get_current_epoch_committee_count(state) + ) % SHARD_COUNT + + +def process_slashings(state: BeaconState) -> None: + """ + Process the slashings. + Note that this function mutates ``state``. + """ + current_epoch = get_current_epoch(state) + active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) + total_balance = get_total_balance(state, active_validator_indices) + + # Compute `total_penalties` + total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] + total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + total_penalties = total_at_end - total_at_start + + for index, validator in enumerate(state.validator_registry): + if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: + penalty = max( + get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, + get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT + ) + decrease_balance(state, index, penalty) + + +def process_exit_queue(state: BeaconState) -> None: + """ + Process the exit queue. + Note that this function mutates ``state``. + """ + def eligible(index): + validator = state.validator_registry[index] + # Filter out dequeued validators + if validator.withdrawable_epoch != FAR_FUTURE_EPOCH: + return False + # Dequeue if the minimum amount of time has passed + else: + return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY + + eligible_indices = filter(eligible, list(range(len(state.validator_registry)))) + # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index + sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) + for dequeues, index in enumerate(sorted_indices): + if dequeues >= MAX_EXIT_DEQUEUES_PER_EPOCH: + break + prepare_validator_for_withdrawal(state, index) + + +def finish_epoch_update(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = current_epoch + 1 + # Set active index root + index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH + state.latest_active_index_roots[index_root_position] = hash_tree_root( + get_active_validator_indices(state.validator_registry, next_epoch + ACTIVATION_EXIT_DELAY) + ) + # Set total slashed balances + state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( + state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] + ) + # Set randao mix + state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) + # Set historical root accumulator + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch( + block_roots=state.latest_block_roots, + state_roots=state.latest_state_roots, + ) + state.historical_roots.append(hash_tree_root(historical_batch)) + # Rotate current/previous epoch attestations + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] + + +def advance_slot(state: BeaconState) -> None: + state.slot += 1 + + +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the parent matches + assert block.previous_block_root == signed_root(state.latest_block_header) + # Save current block as the new latest block + state.latest_block_header = get_temporary_block_header(block) + # Verify proposer is not slashed + proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] + assert not proposer.slashed + # Verify proposer signature + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=signed_root(block), + signature=block.signature, + domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) + ) + + +def process_randao(state: BeaconState, block: BeaconBlock) -> None: + proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] + # Verify that the provided randao value is valid + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=hash_tree_root(get_current_epoch(state)), + signature=block.body.randao_reveal, + domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) + ) + # Mix it in + state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( + xor(get_randao_mix(state, get_current_epoch(state)), + hash(block.body.randao_reveal)) + ) + + +def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: + for eth1_data_vote in state.eth1_data_votes: + # If someone else has already voted for the same hash, add to its counter + if eth1_data_vote.eth1_data == block.body.eth1_data: + eth1_data_vote.vote_count += 1 + return + # If we're seeing this hash for the first time, make a new counter + state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) + + +def process_proposer_slashing(state: BeaconState, + proposer_slashing: ProposerSlashing) -> None: + """ + Process ``ProposerSlashing`` transaction. + Note that this function mutates ``state``. + """ + proposer = state.validator_registry[proposer_slashing.proposer_index] + # Verify that the epoch is the same + assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) + # But the headers are different + assert proposer_slashing.header_1 != proposer_slashing.header_2 + # Check proposer is slashable + assert is_slashable_validator(proposer, get_current_epoch(state)) + # Signatures are valid + for header in (proposer_slashing.header_1, proposer_slashing.header_2): + assert bls_verify( + pubkey=proposer.pubkey, + message_hash=signed_root(header), + signature=header.signature, + domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) + ) + slash_validator(state, proposer_slashing.proposer_index) + + +def process_attester_slashing(state: BeaconState, + attester_slashing: AttesterSlashing) -> None: + """ + Process ``AttesterSlashing`` transaction. + Note that this function mutates ``state``. + """ + attestation1 = attester_slashing.slashable_attestation_1 + attestation2 = attester_slashing.slashable_attestation_2 + # Check that the attestations are conflicting + assert attestation1.data != attestation2.data + assert ( + is_double_vote(attestation1.data, attestation2.data) or + is_surround_vote(attestation1.data, attestation2.data) + ) + assert verify_slashable_attestation(state, attestation1) + assert verify_slashable_attestation(state, attestation2) + slashable_indices = [ + index for index in attestation1.validator_indices + if ( + index in attestation2.validator_indices and + is_slashable_validator(state.validator_registry[index], get_current_epoch(state)) + ) + ] + assert len(slashable_indices) >= 1 + for index in slashable_indices: + slash_validator(state, index) + + +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + """ + Process ``Attestation`` transaction. + Note that this function mutates ``state``. + """ + assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot + assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY + + # Check target epoch, source epoch, and source root + target_epoch = slot_to_epoch(attestation.data.slot) + assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in { + (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), + (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), + } + + # Check crosslink data + assert attestation.data.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] + assert state.latest_crosslinks[attestation.data.shard] in { + attestation.data.previous_crosslink, # Case 1: latest crosslink matches previous crosslink + Crosslink( # Case 2: latest crosslink matches current crosslink + crosslink_data_root=attestation.data.crosslink_data_root, + epoch=min(slot_to_epoch(attestation.data.slot), + attestation.data.previous_crosslink.epoch + MAX_CROSSLINK_EPOCHS) + ), + } + + # Check custody bits [to be generalised in phase 1] + assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) + + # Check aggregate signature [to be generalised in phase 1] + participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) + assert len(participants) != 0 + assert bls_verify( + pubkey=bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in participants]), + message_hash=hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), + signature=attestation.aggregate_signature, + domain=get_domain(state.fork, target_epoch, DOMAIN_ATTESTATION), + ) + + # Cache pending attestation + pending_attestation = PendingAttestation( + data=attestation.data, + aggregation_bitfield=attestation.aggregation_bitfield, + custody_bitfield=attestation.custody_bitfield, + inclusion_slot=state.slot + ) + if target_epoch == get_current_epoch(state): + state.current_epoch_attestations.append(pending_attestation) + else: + state.previous_epoch_attestations.append(pending_attestation) + + +def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: + """ + Process ``VoluntaryExit`` transaction. + Note that this function mutates ``state``. + """ + validator = state.validator_registry[exit.validator_index] + # Verify the validator is active + assert is_active_validator(validator, get_current_epoch(state)) + # Verify the validator has not yet exited + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Verify the validator has not initiated an exit + assert validator.initiated_exit is False + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= exit.epoch + # Verify the validator has been active long enough + assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD + # Verify signature + assert bls_verify( + pubkey=validator.pubkey, + message_hash=signed_root(exit), + signature=exit.signature, + domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) + ) + # Initiate exit + initiate_validator_exit(state, exit.validator_index) + + +def process_transfer(state: BeaconState, transfer: Transfer) -> None: + """ + Process ``Transfer`` transaction. + Note that this function mutates ``state``. + """ + # Verify the amount and fee aren't individually too big (for anti-overflow purposes) + assert get_balance(state, transfer.sender) >= max(transfer.amount, transfer.fee) + # Verify that we have enough ETH to send, and that after the transfer the balance will be either + # exactly zero or at least MIN_DEPOSIT_AMOUNT + assert ( + get_balance(state, transfer.sender) == transfer.amount + transfer.fee or + get_balance(state, transfer.sender) >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT + ) + # A transfer is valid in only one slot + assert state.slot == transfer.slot + # Only withdrawn or not-yet-deposited accounts can transfer + assert ( + get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or + state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH + ) + # Verify that the pubkey is valid + assert ( + state.validator_registry[transfer.sender].withdrawal_credentials == + BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] + ) + # Verify that the signature is valid + assert bls_verify( + pubkey=transfer.pubkey, + message_hash=signed_root(transfer), + signature=transfer.signature, + domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) + ) + # Process the transfer + decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) + increase_balance(state, transfer.recipient, transfer.amount) + increase_balance(state, get_beacon_proposer_index(state, state.slot), transfer.fee) + + +def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: + assert block.state_root == hash_tree_root(state) + +# Monkey patch validator get committee code +_compute_committee = compute_committee +committee_cache = {} + + +def compute_committee(validator_indices: List[ValidatorIndex], + seed: Bytes32, + index: int, + total_committees: int) -> List[ValidatorIndex]: + + param_hash = (hash_tree_root(validator_indices), seed, index, total_committees) + + if param_hash in committee_cache: + # print("Cache hit, epoch={0}".format(epoch)) + return committee_cache[param_hash] + else: + # print("Cache miss, epoch={0}".format(epoch)) + ret = _compute_committee(validator_indices, seed, index, total_committees) + committee_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} + + +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + \ No newline at end of file diff --git a/test_libs/pyspec/phase0/state_transition.py b/test_libs/pyspec/eth2/phase0/state_transition.py similarity index 97% rename from test_libs/pyspec/phase0/state_transition.py rename to test_libs/pyspec/eth2/phase0/state_transition.py index cfd941c420..f2fb68f6ba 100644 --- a/test_libs/pyspec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2/phase0/state_transition.py @@ -1,12 +1,10 @@ from . import spec -from typing import ( # noqa: F401 +from typing import ( Any, Callable, - List, - NewType, - Tuple, + List ) from .spec import ( diff --git a/test_libs/pyspec/eth2/utils/__init__.py b/test_libs/pyspec/eth2/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/utils/bls_stub.py b/test_libs/pyspec/eth2/utils/bls_stub.py similarity index 100% rename from test_libs/pyspec/utils/bls_stub.py rename to test_libs/pyspec/eth2/utils/bls_stub.py diff --git a/test_libs/pyspec/utils/hash_function.py b/test_libs/pyspec/eth2/utils/hash_function.py similarity index 100% rename from test_libs/pyspec/utils/hash_function.py rename to test_libs/pyspec/eth2/utils/hash_function.py diff --git a/test_libs/pyspec/utils/merkle_minimal.py b/test_libs/pyspec/eth2/utils/merkle_minimal.py similarity index 100% rename from test_libs/pyspec/utils/merkle_minimal.py rename to test_libs/pyspec/eth2/utils/merkle_minimal.py diff --git a/test_libs/pyspec/utils/minimal_ssz.py b/test_libs/pyspec/eth2/utils/minimal_ssz.py similarity index 100% rename from test_libs/pyspec/utils/minimal_ssz.py rename to test_libs/pyspec/eth2/utils/minimal_ssz.py diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 5d121a2633..64ae482fd7 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -3,7 +3,7 @@ setup( name='pyspec', version='1.0', - packages=['debug', 'utils', 'phase0'], + packages=['eth2'], install_requires=[ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", From ec4d41e15da01e4da6626d41e11fe034a8f87476 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 02:30:47 +0800 Subject: [PATCH 08/61] fix config, work on py_tests --- .gitignore | 1 + Makefile | 2 +- py_tests/README.md | 23 +++++++++++++++++++ .../test_process_attestation.py | 9 ++++---- .../test_process_block_header.py | 4 ++-- .../block_processing/test_process_deposit.py | 6 ++--- .../test_process_proposer_slashing.py | 6 ++--- .../block_processing/test_voluntary_exit.py | 6 ++--- py_tests/phase0/conftest.py | 4 ++-- py_tests/phase0/helpers.py | 8 +++---- py_tests/phase0/test_sanity.py | 12 +++++----- py_tests/requirements.txt | 1 + test_generators/README.md | 2 +- 13 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 py_tests/README.md diff --git a/.gitignore b/.gitignore index 816ecfa269..16576a6349 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ build/ output/ yaml_tests/ +.pytest_cache diff --git a/Makefile b/Makefile index 3812a8255d..84dbd0c264 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ define build_yaml_tests # Activate the venv, this is where dependencies are installed for the generator . $(GENERATOR_VENVS_DIR)$(1)bin/activate # Install all the necessary requirements - pip3 install -r $(GENERATOR_DIR)$(1)requirements.txt --user + pip3 install -r $(GENERATOR_DIR)$(1)requirements.txt # Run the generator. The generator is assumed to have an "main.py" file. # We output to the tests dir (generator program should accept a "-p " argument. diff --git a/py_tests/README.md b/py_tests/README.md new file mode 100644 index 0000000000..659e737f14 --- /dev/null +++ b/py_tests/README.md @@ -0,0 +1,23 @@ +# ETH 2.0 py-tests + +These tests are not intended for client-consumption. +These tests are sanity tests, to verify if the spec itself is consistent. + +There are ideas to port these tests to the YAML test suite, + but we are still looking for inputs on how this should work. + +## How to run tests + +From within the py_tests folder: + +Install dependencies: +```bash +python3 -m venv venv +. py_tests/venv/bin/activate +pip3 install -r requirements.txt +``` + +Run the tests: +``` +pytest -m minimal_config . +``` diff --git a/py_tests/phase0/block_processing/test_process_attestation.py b/py_tests/phase0/block_processing/test_process_attestation.py index 08cab11ff1..f76ca77d8c 100644 --- a/py_tests/phase0/block_processing/test_process_attestation.py +++ b/py_tests/phase0/block_processing/test_process_attestation.py @@ -1,18 +1,17 @@ from copy import deepcopy import pytest -import build.phase0.spec as spec +import eth2.phase0.spec as spec -from build.phase0.state_transition import ( +from eth2.phase0.state_transition import ( state_transition, ) -from build.phase0.spec import ( - ZERO_HASH, +from eth2.phase0.spec import ( get_current_epoch, process_attestation, slot_to_epoch, ) -from tests.phase0.helpers import ( +from ..helpers import ( build_empty_block_for_next_slot, get_valid_attestation, ) diff --git a/py_tests/phase0/block_processing/test_process_block_header.py b/py_tests/phase0/block_processing/test_process_block_header.py index 4981b656cb..ef902bb727 100644 --- a/py_tests/phase0/block_processing/test_process_block_header.py +++ b/py_tests/phase0/block_processing/test_process_block_header.py @@ -2,13 +2,13 @@ import pytest -from build.phase0.spec import ( +from eth2.phase0.spec import ( get_beacon_proposer_index, cache_state, advance_slot, process_block_header, ) -from tests.phase0.helpers import ( +from ..helpers import ( build_empty_block_for_next_slot, ) diff --git a/py_tests/phase0/block_processing/test_process_deposit.py b/py_tests/phase0/block_processing/test_process_deposit.py index 0726dddef0..19467984e3 100644 --- a/py_tests/phase0/block_processing/test_process_deposit.py +++ b/py_tests/phase0/block_processing/test_process_deposit.py @@ -1,14 +1,14 @@ from copy import deepcopy import pytest -import build.phase0.spec as spec +import eth2.phase0.spec as spec -from build.phase0.spec import ( +from eth2.phase0.spec import ( get_balance, ZERO_HASH, process_deposit, ) -from tests.phase0.helpers import ( +from ..helpers import ( build_deposit, privkeys, pubkeys, diff --git a/py_tests/phase0/block_processing/test_process_proposer_slashing.py b/py_tests/phase0/block_processing/test_process_proposer_slashing.py index 467d2164ba..761624c256 100644 --- a/py_tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/py_tests/phase0/block_processing/test_process_proposer_slashing.py @@ -1,13 +1,13 @@ from copy import deepcopy import pytest -import build.phase0.spec as spec -from build.phase0.spec import ( +import eth2.phase0.spec as spec +from eth2.phase0.spec import ( get_balance, get_current_epoch, process_proposer_slashing, ) -from tests.phase0.helpers import ( +from ..helpers import ( get_valid_proposer_slashing, ) diff --git a/py_tests/phase0/block_processing/test_voluntary_exit.py b/py_tests/phase0/block_processing/test_voluntary_exit.py index 6adc814644..45a8af1bbc 100644 --- a/py_tests/phase0/block_processing/test_voluntary_exit.py +++ b/py_tests/phase0/block_processing/test_voluntary_exit.py @@ -1,14 +1,14 @@ from copy import deepcopy import pytest -import build.phase0.spec as spec +import eth2.phase0.spec as spec -from build.phase0.spec import ( +from eth2.phase0.spec import ( get_active_validator_indices, get_current_epoch, process_voluntary_exit, ) -from tests.phase0.helpers import ( +from ..helpers import ( build_voluntary_exit, pubkey_to_privkey, ) diff --git a/py_tests/phase0/conftest.py b/py_tests/phase0/conftest.py index 36a0879410..0def66ad6e 100644 --- a/py_tests/phase0/conftest.py +++ b/py_tests/phase0/conftest.py @@ -1,8 +1,8 @@ import pytest -from build.phase0 import spec +from eth2.phase0 import spec -from tests.phase0.helpers import ( +from .helpers import ( create_genesis_state, ) diff --git a/py_tests/phase0/helpers.py b/py_tests/phase0/helpers.py index d7f4ae6e8f..3be231ee28 100644 --- a/py_tests/phase0/helpers.py +++ b/py_tests/phase0/helpers.py @@ -2,9 +2,9 @@ from py_ecc import bls -import build.phase0.spec as spec -from build.phase0.utils.minimal_ssz import signed_root -from build.phase0.spec import ( +import eth2.phase0.spec as spec +from eth2.utils.minimal_ssz import signed_root +from eth2.phase0.spec import ( # constants EMPTY_SIGNATURE, ZERO_HASH, @@ -31,7 +31,7 @@ verify_merkle_branch, hash, ) -from build.phase0.utils.merkle_minimal import ( +from eth2.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/py_tests/phase0/test_sanity.py b/py_tests/phase0/test_sanity.py index 3b4497ca52..0af65ad51b 100644 --- a/py_tests/phase0/test_sanity.py +++ b/py_tests/phase0/test_sanity.py @@ -3,10 +3,10 @@ import pytest from py_ecc import bls -import build.phase0.spec as spec +import eth2.phase0.spec as spec -from build.phase0.utils.minimal_ssz import signed_root -from build.phase0.spec import ( +from eth2.utils.minimal_ssz import signed_root +from eth2.phase0.spec import ( # constants EMPTY_SIGNATURE, ZERO_HASH, @@ -27,15 +27,15 @@ verify_merkle_branch, hash, ) -from build.phase0.state_transition import ( +from eth2.phase0.state_transition import ( state_transition, ) -from build.phase0.utils.merkle_minimal import ( +from eth2.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, ) -from tests.phase0.helpers import ( +from .helpers import ( build_deposit_data, build_empty_block_for_next_slot, force_registry_change_at_next_epoch, diff --git a/py_tests/requirements.txt b/py_tests/requirements.txt index 9145e951e5..d18b29127e 100644 --- a/py_tests/requirements.txt +++ b/py_tests/requirements.txt @@ -4,3 +4,4 @@ oyaml==0.7 pycryptodome==3.7.3 py_ecc>=1.6.0 pytest>=3.6,<3.7 +../pyspec diff --git a/test_generators/README.md b/test_generators/README.md index 2d6160c03c..61dc960637 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -62,7 +62,7 @@ eth-utils==1.4.1 Install all the necessary requirements (re-run when you add more): ```bash -pip3 install -r requirements.txt --user +pip3 install -r requirements.txt ``` And write your initial test generator, extending the base generator: From 3f6d6535ada35fd648f7ed6fad084c080f8af037 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 22:04:51 +0800 Subject: [PATCH 09/61] Fix pyspec package structure --- .gitignore | 3 + test_libs/pyspec/eth2/__init__.py | 0 test_libs/pyspec/eth2/phase0/spec.py | 1692 -------------------------- test_libs/pyspec/setup.py | 2 +- 4 files changed, 4 insertions(+), 1693 deletions(-) delete mode 100644 test_libs/pyspec/eth2/__init__.py delete mode 100644 test_libs/pyspec/eth2/phase0/spec.py diff --git a/.gitignore b/.gitignore index 16576a6349..b12e536ffe 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ output/ yaml_tests/ .pytest_cache + +# Dynamically built from Markdown spec +test_libs/pyspec/eth2/phase0/spec.py diff --git a/test_libs/pyspec/eth2/__init__.py b/test_libs/pyspec/eth2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test_libs/pyspec/eth2/phase0/spec.py b/test_libs/pyspec/eth2/phase0/spec.py deleted file mode 100644 index 32dc8e7438..0000000000 --- a/test_libs/pyspec/eth2/phase0/spec.py +++ /dev/null @@ -1,1692 +0,0 @@ -from eth2.utils.minimal_ssz import * -from eth2.utils.bls_stub import * -def int_to_bytes1(x): return x.to_bytes(1, 'little') -def int_to_bytes2(x): return x.to_bytes(2, 'little') -def int_to_bytes3(x): return x.to_bytes(3, 'little') -def int_to_bytes4(x): return x.to_bytes(4, 'little') -def int_to_bytes8(x): return x.to_bytes(8, 'little') -def int_to_bytes32(x): return x.to_bytes(32, 'little') -def int_to_bytes48(x): return x.to_bytes(48, 'little') -def int_to_bytes96(x): return x.to_bytes(96, 'little') - - -SLOTS_PER_EPOCH = 64 -def slot_to_epoch(x): return x // SLOTS_PER_EPOCH - - -from typing import ( - Any, - Callable, - List, - NewType, - Tuple, -) - - -Slot = NewType('Slot', int) # uint64 -Epoch = NewType('Epoch', int) # uint64 -Shard = NewType('Shard', int) # uint64 -ValidatorIndex = NewType('ValidatorIndex', int) # uint64 -Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 -Any = None -Store = None - -SHARD_COUNT = 2**10 -TARGET_COMMITTEE_SIZE = 2**7 -MAX_BALANCE_CHURN_QUOTIENT = 2**5 -MAX_SLASHABLE_ATTESTATION_PARTICIPANTS = 2**12 -MAX_EXIT_DEQUEUES_PER_EPOCH = 2**2 -SHUFFLE_ROUND_COUNT = 90 -DEPOSIT_CONTRACT_ADDRESS = 0x1234567890123567890123456789012357890 -DEPOSIT_CONTRACT_TREE_DEPTH = 2**5 -MIN_DEPOSIT_AMOUNT = 2**0 * 10**9 -MAX_DEPOSIT_AMOUNT = 2**5 * 10**9 -EJECTION_BALANCE = 2**4 * 10**9 -HIGH_BALANCE_INCREMENT = 2**0 * 10**9 -GENESIS_FORK_VERSION = int_to_bytes4(0) -GENESIS_SLOT = 2**32 -GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) -GENESIS_START_SHARD = 0 -FAR_FUTURE_EPOCH = 2**64 - 1 -ZERO_HASH = int_to_bytes32(0) -EMPTY_SIGNATURE = int_to_bytes96(0) -BLS_WITHDRAWAL_PREFIX_BYTE = int_to_bytes1(0) -SECONDS_PER_SLOT = 6 -MIN_ATTESTATION_INCLUSION_DELAY = 2**2 -SLOTS_PER_EPOCH = 2**6 -MIN_SEED_LOOKAHEAD = 2**0 -ACTIVATION_EXIT_DELAY = 2**2 -EPOCHS_PER_ETH1_VOTING_PERIOD = 2**4 -SLOTS_PER_HISTORICAL_ROOT = 2**13 -MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8 -PERSISTENT_COMMITTEE_PERIOD = 2**11 -MAX_CROSSLINK_EPOCHS = 2**6 -LATEST_RANDAO_MIXES_LENGTH = 2**13 -LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 2**13 -LATEST_SLASHED_EXIT_LENGTH = 2**13 -BASE_REWARD_QUOTIENT = 2**5 -WHISTLEBLOWER_REWARD_QUOTIENT = 2**9 -ATTESTATION_INCLUSION_REWARD_QUOTIENT = 2**3 -INACTIVITY_PENALTY_QUOTIENT = 2**24 -MIN_PENALTY_QUOTIENT = 2**5 -MAX_PROPOSER_SLASHINGS = 2**4 -MAX_ATTESTER_SLASHINGS = 2**0 -MAX_ATTESTATIONS = 2**7 -MAX_DEPOSITS = 2**4 -MAX_VOLUNTARY_EXITS = 2**4 -MAX_TRANSFERS = 2**4 -DOMAIN_BEACON_BLOCK = 0 -DOMAIN_RANDAO = 1 -DOMAIN_ATTESTATION = 2 -DOMAIN_DEPOSIT = 3 -DOMAIN_VOLUNTARY_EXIT = 4 -DOMAIN_TRANSFER = 5 -Fork = SSZType({ - # Previous fork version - 'previous_version': 'bytes4', - # Current fork version - 'current_version': 'bytes4', - # Fork epoch number - 'epoch': 'uint64', -}) -Crosslink = SSZType({ - # Epoch number - 'epoch': 'uint64', - # Shard data since the previous crosslink - 'crosslink_data_root': 'bytes32', -}) -Eth1Data = SSZType({ - # Root of the deposit tree - 'deposit_root': 'bytes32', - # Total number of deposits - 'deposit_count': 'uint64', - # Block hash - 'block_hash': 'bytes32', -}) -Eth1DataVote = SSZType({ - # Data being voted for - 'eth1_data': Eth1Data, - # Vote count - 'vote_count': 'uint64', -}) -AttestationData = SSZType({ - # LMD GHOST vote - 'slot': 'uint64', - 'beacon_block_root': 'bytes32', - - # FFG vote - 'source_epoch': 'uint64', - 'source_root': 'bytes32', - 'target_root': 'bytes32', - - # Crosslink vote - 'shard': 'uint64', - 'previous_crosslink': Crosslink, - 'crosslink_data_root': 'bytes32', -}) -AttestationDataAndCustodyBit = SSZType({ - # Attestation data - 'data': AttestationData, - # Custody bit - 'custody_bit': 'bool', -}) -SlashableAttestation = SSZType({ - # Validator indices - 'validator_indices': ['uint64'], - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # Aggregate signature - 'aggregate_signature': 'bytes96', -}) -DepositData = SSZType({ - # BLS pubkey - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # Amount in Gwei - 'amount': 'uint64', - # Container self-signature - 'proof_of_possession': 'bytes96', -}) -BeaconBlockHeader = SSZType({ - 'slot': 'uint64', - 'previous_block_root': 'bytes32', - 'state_root': 'bytes32', - 'block_body_root': 'bytes32', - 'signature': 'bytes96', -}) -Validator = SSZType({ - # BLS public key - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # Epoch when validator activated - 'activation_epoch': 'uint64', - # Epoch when validator exited - 'exit_epoch': 'uint64', - # Epoch when validator is eligible to withdraw - 'withdrawable_epoch': 'uint64', - # Did the validator initiate an exit - 'initiated_exit': 'bool', - # Was the validator slashed - 'slashed': 'bool', - # Rounded balance - 'high_balance': 'uint64' -}) -PendingAttestation = SSZType({ - # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # Inclusion slot - 'inclusion_slot': 'uint64', -}) -HistoricalBatch = SSZType({ - # Block roots - 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - # State roots - 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], -}) -ProposerSlashing = SSZType({ - # Proposer index - 'proposer_index': 'uint64', - # First block header - 'header_1': BeaconBlockHeader, - # Second block header - 'header_2': BeaconBlockHeader, -}) -AttesterSlashing = SSZType({ - # First slashable attestation - 'slashable_attestation_1': SlashableAttestation, - # Second slashable attestation - 'slashable_attestation_2': SlashableAttestation, -}) -Attestation = SSZType({ - # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', - # Attestation data - 'data': AttestationData, - # Custody bitfield - 'custody_bitfield': 'bytes', - # BLS aggregate signature - 'aggregate_signature': 'bytes96', -}) -Deposit = SSZType({ - # Branch in the deposit tree - 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], - # Index in the deposit tree - 'index': 'uint64', - # Data - 'data': DepositData, -}) -VoluntaryExit = SSZType({ - # Minimum epoch for processing exit - 'epoch': 'uint64', - # Index of the exiting validator - 'validator_index': 'uint64', - # Validator signature - 'signature': 'bytes96', -}) -Transfer = SSZType({ - # Sender index - 'sender': 'uint64', - # Recipient index - 'recipient': 'uint64', - # Amount in Gwei - 'amount': 'uint64', - # Fee in Gwei for block proposer - 'fee': 'uint64', - # Inclusion slot - 'slot': 'uint64', - # Sender withdrawal pubkey - 'pubkey': 'bytes48', - # Sender signature - 'signature': 'bytes96', -}) -BeaconBlockBody = SSZType({ - 'randao_reveal': 'bytes96', - 'eth1_data': Eth1Data, - 'proposer_slashings': [ProposerSlashing], - 'attester_slashings': [AttesterSlashing], - 'attestations': [Attestation], - 'deposits': [Deposit], - 'voluntary_exits': [VoluntaryExit], - 'transfers': [Transfer], -}) -BeaconBlock = SSZType({ - # Header - 'slot': 'uint64', - 'previous_block_root': 'bytes32', - 'state_root': 'bytes32', - 'body': BeaconBlockBody, - 'signature': 'bytes96', -}) -BeaconState = SSZType({ - # Misc - 'slot': 'uint64', - 'genesis_time': 'uint64', - 'fork': Fork, # For versioning hard forks - - # Validator registry - 'validator_registry': [Validator], - 'balances': ['uint64'], - 'validator_registry_update_epoch': 'uint64', - - # Randomness and committees - 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'latest_start_shard': 'uint64', - - # Finality - 'previous_epoch_attestations': [PendingAttestation], - 'current_epoch_attestations': [PendingAttestation], - 'previous_justified_epoch': 'uint64', - 'current_justified_epoch': 'uint64', - 'previous_justified_root': 'bytes32', - 'current_justified_root': 'bytes32', - 'justification_bitfield': 'uint64', - 'finalized_epoch': 'uint64', - 'finalized_root': 'bytes32', - - # Recent state - 'latest_crosslinks': [Crosslink, SHARD_COUNT], - 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period - 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily - 'historical_roots': ['bytes32'], - - # Ethereum 1.0 chain data - 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1DataVote], - 'deposit_index': 'uint64', -}) - - -def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: - return bytes(a ^ b for a, b in zip(bytes1, bytes2)) - - -def get_temporary_block_header(block: BeaconBlock) -> BeaconBlockHeader: - """ - Return the block header corresponding to a block with ``state_root`` set to ``ZERO_HASH``. - """ - return BeaconBlockHeader( - slot=block.slot, - previous_block_root=block.previous_block_root, - state_root=ZERO_HASH, - block_body_root=hash_tree_root(block.body), - # signed_root(block) is used for block id purposes so signature is a stub - signature=EMPTY_SIGNATURE, - ) - - -def slot_to_epoch(slot: Slot) -> Epoch: - """ - Return the epoch number of the given ``slot``. - """ - return slot // SLOTS_PER_EPOCH - - -def get_previous_epoch(state: BeaconState) -> Epoch: - """` - Return the previous epoch of the given ``state``. - """ - return get_current_epoch(state) - 1 - - -def get_current_epoch(state: BeaconState) -> Epoch: - """ - Return the current epoch of the given ``state``. - """ - return slot_to_epoch(state.slot) - - -def get_epoch_start_slot(epoch: Epoch) -> Slot: - """ - Return the starting slot of the given ``epoch``. - """ - return epoch * SLOTS_PER_EPOCH - - -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is active. - """ - return validator.activation_epoch <= epoch < validator.exit_epoch - - -def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is slashable. - """ - return ( - validator.activation_epoch <= epoch < validator.withdrawable_epoch and - validator.slashed is False - ) - - -def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: - """ - Get indices of active validators from ``validators``. - """ - return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] - - -def get_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: - """ - Return the balance for a validator with the given ``index``. - """ - return state.balances[index] - - -def set_balance(state: BeaconState, index: ValidatorIndex, balance: Gwei) -> None: - """ - Set the balance for a validator with the given ``index`` in both ``BeaconState`` - and validator's rounded balance ``high_balance``. - """ - validator = state.validator_registry[index] - HALF_INCREMENT = HIGH_BALANCE_INCREMENT // 2 - if validator.high_balance > balance or validator.high_balance + 3 * HALF_INCREMENT < balance: - validator.high_balance = balance - balance % HIGH_BALANCE_INCREMENT - state.balances[index] = balance - - -def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: - """ - Increase the balance for a validator with the given ``index`` by ``delta``. - """ - set_balance(state, index, get_balance(state, index) + delta) - - -def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: - """ - Decrease the balance for a validator with the given ``index`` by ``delta``. - Set to ``0`` when underflow. - """ - current_balance = get_balance(state, index) - set_balance(state, index, current_balance - delta if current_balance >= delta else 0) - - -def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: - """ - Return `p(index)` in a pseudorandom permutation `p` of `0...list_size - 1` with ``seed`` as entropy. - - Utilizes 'swap or not' shuffling found in - https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf - See the 'generalized domain' algorithm on page 3. - """ - assert index < list_size - assert list_size <= 2**40 - - for round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size - flip = (pivot - index) % list_size - position = max(index, flip) - source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) - byte = source[(position % 256) // 8] - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return index - - -def get_split_offset(list_size: int, chunks: int, index: int) -> int: - """ - Returns a value such that for a list L, chunk count k and index i, - split(L, k)[i] == L[get_split_offset(len(L), k, i): get_split_offset(len(L), k, i+1)] - """ - return (list_size * index) // chunks - - -def get_epoch_committee_count(active_validator_count: int) -> int: - """ - Return the number of committees in one epoch. - """ - return max( - 1, - min( - SHARD_COUNT // SLOTS_PER_EPOCH, - active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - ) - ) * SLOTS_PER_EPOCH - - -def compute_committee(validator_indices: List[ValidatorIndex], - seed: Bytes32, - index: int, - total_committees: int) -> List[ValidatorIndex]: - """ - Return the ``index``'th shuffled committee out of a total ``total_committees`` - using ``validator_indices`` and ``seed``. - """ - start_offset = get_split_offset(len(validator_indices), total_committees, index) - end_offset = get_split_offset(len(validator_indices), total_committees, index + 1) - return [ - validator_indices[get_permuted_index(i, len(validator_indices), seed)] - for i in range(start_offset, end_offset) - ] - - -def get_current_epoch_committee_count(state: BeaconState) -> int: - """ - Return the number of committees in the current epoch of the given ``state``. - """ - current_active_validators = get_active_validator_indices( - state.validator_registry, - get_current_epoch(state), - ) - return get_epoch_committee_count(len(current_active_validators)) - - -def get_crosslink_committees_at_slot(state: BeaconState, - slot: Slot) -> List[Tuple[List[ValidatorIndex], Shard]]: - """ - Return the list of ``(committee, shard)`` tuples for the ``slot``. - """ - epoch = slot_to_epoch(slot) - current_epoch = get_current_epoch(state) - previous_epoch = get_previous_epoch(state) - next_epoch = current_epoch + 1 - - assert previous_epoch <= epoch <= next_epoch - indices = get_active_validator_indices( - state.validator_registry, - epoch, - ) - committees_per_epoch = get_epoch_committee_count(len(indices)) - - if epoch == current_epoch: - start_shard = state.latest_start_shard - elif epoch == previous_epoch: - start_shard = (state.latest_start_shard - committees_per_epoch) % SHARD_COUNT - elif epoch == next_epoch: - current_epoch_committees = get_current_epoch_committee_count(state) - start_shard = (state.latest_start_shard + current_epoch_committees) % SHARD_COUNT - - committees_per_slot = committees_per_epoch // SLOTS_PER_EPOCH - offset = slot % SLOTS_PER_EPOCH - slot_start_shard = (start_shard + committees_per_slot * offset) % SHARD_COUNT - seed = generate_seed(state, epoch) - - return [ - ( - compute_committee(indices, seed, committees_per_slot * offset + i, committees_per_epoch), - (slot_start_shard + i) % SHARD_COUNT, - ) - for i in range(committees_per_slot) - ] - - -def get_block_root(state: BeaconState, - slot: Slot) -> Bytes32: - """ - Return the block root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] - - -def get_state_root(state: BeaconState, - slot: Slot) -> Bytes32: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % SLOTS_PER_HISTORICAL_ROOT] - - -def get_randao_mix(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Return the randao mix at a recent ``epoch``. - """ - assert get_current_epoch(state) - LATEST_RANDAO_MIXES_LENGTH < epoch <= get_current_epoch(state) - return state.latest_randao_mixes[epoch % LATEST_RANDAO_MIXES_LENGTH] - - -def get_active_index_root(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Return the index root at a recent ``epoch``. - """ - assert get_current_epoch(state) - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY < epoch <= get_current_epoch(state) + ACTIVATION_EXIT_DELAY - return state.latest_active_index_roots[epoch % LATEST_ACTIVE_INDEX_ROOTS_LENGTH] - - -def generate_seed(state: BeaconState, - epoch: Epoch) -> Bytes32: - """ - Generate a seed for the given ``epoch``. - """ - return hash( - get_randao_mix(state, epoch - MIN_SEED_LOOKAHEAD) + - get_active_index_root(state, epoch) + - int_to_bytes32(epoch) - ) - - -def get_beacon_proposer_index(state: BeaconState, - slot: Slot) -> ValidatorIndex: - """ - Return the beacon proposer index for the ``slot``. - Due to proposer selection being based upon the validator balances during - the epoch in question, this can only be run for the current epoch. - """ - current_epoch = get_current_epoch(state) - assert slot_to_epoch(slot) == current_epoch - - first_committee, _ = get_crosslink_committees_at_slot(state, slot)[0] - i = 0 - while True: - rand_byte = hash( - generate_seed(state, current_epoch) + - int_to_bytes8(i // 32) - )[i % 32] - candidate = first_committee[(current_epoch + i) % len(first_committee)] - if get_effective_balance(state, candidate) * 256 > MAX_DEPOSIT_AMOUNT * rand_byte: - return candidate - i += 1 - - -def verify_merkle_branch(leaf: Bytes32, proof: List[Bytes32], depth: int, index: int, root: Bytes32) -> bool: - """ - Verify that the given ``leaf`` is on the merkle branch ``proof`` - starting with the given ``root``. - """ - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(proof[i] + value) - else: - value = hash(value + proof[i]) - return value == root - - -def get_attestation_participants(state: BeaconState, - attestation_data: AttestationData, - bitfield: bytes) -> List[ValidatorIndex]: - """ - Return the participant indices corresponding to ``attestation_data`` and ``bitfield``. - """ - # Find the committee in the list with the desired shard - crosslink_committees = get_crosslink_committees_at_slot(state, attestation_data.slot) - - assert attestation_data.shard in [shard for _, shard in crosslink_committees] - crosslink_committee = [committee for committee, shard in crosslink_committees if shard == attestation_data.shard][0] - - assert verify_bitfield(bitfield, len(crosslink_committee)) - - # Find the participating attesters in the committee - participants = [] - for i, validator_index in enumerate(crosslink_committee): - aggregation_bit = get_bitfield_bit(bitfield, i) - if aggregation_bit == 0b1: - participants.append(validator_index) - return participants - - -def bytes_to_int(data: bytes) -> int: - return int.from_bytes(data, 'little') - - -def get_effective_balance(state: BeaconState, index: ValidatorIndex) -> Gwei: - """ - Return the effective balance (also known as "balance at stake") for a validator with the given ``index``. - """ - return min(get_balance(state, index), MAX_DEPOSIT_AMOUNT) - - -def get_total_balance(state: BeaconState, validators: List[ValidatorIndex]) -> Gwei: - """ - Return the combined effective balance of an array of ``validators``. - """ - return sum([get_effective_balance(state, i) for i in validators]) - - -def get_fork_version(fork: Fork, - epoch: Epoch) -> bytes: - """ - Return the fork version of the given ``epoch``. - """ - if epoch < fork.epoch: - return fork.previous_version - else: - return fork.current_version - - -def get_domain(fork: Fork, - epoch: Epoch, - domain_type: int) -> int: - """ - Get the domain number that represents the fork meta and signature domain. - """ - return bytes_to_int(get_fork_version(fork, epoch) + int_to_bytes4(domain_type)) - - -def get_bitfield_bit(bitfield: bytes, i: int) -> int: - """ - Extract the bit in ``bitfield`` at position ``i``. - """ - return (bitfield[i // 8] >> (i % 8)) % 2 - - -def verify_bitfield(bitfield: bytes, committee_size: int) -> bool: - """ - Verify ``bitfield`` against the ``committee_size``. - """ - if len(bitfield) != (committee_size + 7) // 8: - return False - - # Check `bitfield` is padded with zero bits only - for i in range(committee_size, len(bitfield) * 8): - if get_bitfield_bit(bitfield, i) == 0b1: - return False - - return True - - -def verify_slashable_attestation(state: BeaconState, slashable_attestation: SlashableAttestation) -> bool: - """ - Verify validity of ``slashable_attestation`` fields. - """ - if slashable_attestation.custody_bitfield != b'\x00' * len(slashable_attestation.custody_bitfield): # [TO BE REMOVED IN PHASE 1] - return False - - if not (1 <= len(slashable_attestation.validator_indices) <= MAX_SLASHABLE_ATTESTATION_PARTICIPANTS): - return False - - for i in range(len(slashable_attestation.validator_indices) - 1): - if slashable_attestation.validator_indices[i] >= slashable_attestation.validator_indices[i + 1]: - return False - - if not verify_bitfield(slashable_attestation.custody_bitfield, len(slashable_attestation.validator_indices)): - return False - - custody_bit_0_indices = [] - custody_bit_1_indices = [] - for i, validator_index in enumerate(slashable_attestation.validator_indices): - if get_bitfield_bit(slashable_attestation.custody_bitfield, i) == 0b0: - custody_bit_0_indices.append(validator_index) - else: - custody_bit_1_indices.append(validator_index) - - return bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=slashable_attestation.data, custody_bit=0b1)), - ], - signature=slashable_attestation.aggregate_signature, - domain=get_domain(state.fork, slot_to_epoch(slashable_attestation.data.slot), DOMAIN_ATTESTATION), - ) - - -def is_double_vote(attestation_data_1: AttestationData, - attestation_data_2: AttestationData) -> bool: - """ - Check if ``attestation_data_1`` and ``attestation_data_2`` have the same target. - """ - target_epoch_1 = slot_to_epoch(attestation_data_1.slot) - target_epoch_2 = slot_to_epoch(attestation_data_2.slot) - return target_epoch_1 == target_epoch_2 - - -def is_surround_vote(attestation_data_1: AttestationData, - attestation_data_2: AttestationData) -> bool: - """ - Check if ``attestation_data_1`` surrounds ``attestation_data_2``. - """ - source_epoch_1 = attestation_data_1.source_epoch - source_epoch_2 = attestation_data_2.source_epoch - target_epoch_1 = slot_to_epoch(attestation_data_1.slot) - target_epoch_2 = slot_to_epoch(attestation_data_2.slot) - - return source_epoch_1 < source_epoch_2 and target_epoch_2 < target_epoch_1 - - -def integer_squareroot(n: int) -> int: - """ - The largest integer ``x`` such that ``x**2`` is less than or equal to ``n``. - """ - assert n >= 0 - x = n - y = (x + 1) // 2 - while y < x: - x = y - y = (x + n // x) // 2 - return x - - -def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: - """ - Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. - """ - return epoch + 1 + ACTIVATION_EXIT_DELAY - - -def process_deposit(state: BeaconState, deposit: Deposit) -> None: - """ - Process a deposit from Ethereum 1.0. - Note that this function mutates ``state``. - """ - # Deposits must be processed in order - assert deposit.index == state.deposit_index - - # Verify the Merkle branch - merkle_branch_is_valid = verify_merkle_branch( - leaf=hash(serialize(deposit.data)), # 48 + 32 + 8 + 96 = 184 bytes serialization - proof=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=deposit.index, - root=state.latest_eth1_data.deposit_root, - ) - assert merkle_branch_is_valid - - # Increment the next deposit index we are expecting. Note that this - # needs to be done here because while the deposit contract will never - # create an invalid Merkle branch, it may admit an invalid deposit - # object, and we need to be able to skip over it - state.deposit_index += 1 - - validator_pubkeys = [v.pubkey for v in state.validator_registry] - pubkey = deposit.data.pubkey - amount = deposit.data.amount - - if pubkey not in validator_pubkeys: - # Verify the proof of possession - proof_is_valid = bls_verify( - pubkey=pubkey, - message_hash=signed_root(deposit.data), - signature=deposit.data.proof_of_possession, - domain=get_domain( - state.fork, - get_current_epoch(state), - DOMAIN_DEPOSIT, - ) - ) - if not proof_is_valid: - return - - # Add new validator - validator = Validator( - pubkey=pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - initiated_exit=False, - slashed=False, - high_balance=0 - ) - - # Note: In phase 2 registry indices that have been withdrawn for a long time will be recycled. - state.validator_registry.append(validator) - state.balances.append(0) - set_balance(state, len(state.validator_registry) - 1, amount) - else: - # Increase balance by deposit amount - index = validator_pubkeys.index(pubkey) - increase_balance(state, index, amount) - - -def activate_validator(state: BeaconState, index: ValidatorIndex, is_genesis: bool) -> None: - """ - Activate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - - validator.activation_epoch = GENESIS_EPOCH if is_genesis else get_delayed_activation_exit_epoch(get_current_epoch(state)) - - -def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: - """ - Initiate the validator of the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - validator.initiated_exit = True - - -def exit_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Exit the validator with the given ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - - # Update validator exit epoch if not previously exited - if validator.exit_epoch == FAR_FUTURE_EPOCH: - validator.exit_epoch = get_delayed_activation_exit_epoch(get_current_epoch(state)) - - -def slash_validator(state: BeaconState, index: ValidatorIndex) -> None: - """ - Slash the validator with index ``index``. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - exit_validator(state, index) - state.latest_slashed_balances[get_current_epoch(state) % LATEST_SLASHED_EXIT_LENGTH] += get_effective_balance(state, index) - - whistleblower_index = get_beacon_proposer_index(state, state.slot) - whistleblower_reward = get_effective_balance(state, index) // WHISTLEBLOWER_REWARD_QUOTIENT - increase_balance(state, whistleblower_index, whistleblower_reward) - decrease_balance(state, index, whistleblower_reward) - validator.slashed = True - validator.withdrawable_epoch = get_current_epoch(state) + LATEST_SLASHED_EXIT_LENGTH - - -def prepare_validator_for_withdrawal(state: BeaconState, index: ValidatorIndex) -> None: - """ - Set the validator with the given ``index`` as withdrawable - ``MIN_VALIDATOR_WITHDRAWABILITY_DELAY`` after the current epoch. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[index] - validator.withdrawable_epoch = get_current_epoch(state) + MIN_VALIDATOR_WITHDRAWABILITY_DELAY - - -def get_empty_block() -> BeaconBlock: - """ - Get an empty ``BeaconBlock``. - """ - return BeaconBlock( - slot=GENESIS_SLOT, - previous_block_root=ZERO_HASH, - state_root=ZERO_HASH, - body=BeaconBlockBody( - randao_reveal=EMPTY_SIGNATURE, - eth1_data=Eth1Data( - deposit_root=ZERO_HASH, - deposit_count=0, - block_hash=ZERO_HASH, - ), - proposer_slashings=[], - attester_slashings=[], - attestations=[], - deposits=[], - voluntary_exits=[], - transfers=[], - ), - signature=EMPTY_SIGNATURE, - ) - - -def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], - genesis_time: int, - genesis_eth1_data: Eth1Data) -> BeaconState: - """ - Get the genesis ``BeaconState``. - """ - state = BeaconState( - # Misc - slot=GENESIS_SLOT, - genesis_time=genesis_time, - fork=Fork( - previous_version=GENESIS_FORK_VERSION, - current_version=GENESIS_FORK_VERSION, - epoch=GENESIS_EPOCH, - ), - - # Validator registry - validator_registry=[], - balances=[], - validator_registry_update_epoch=GENESIS_EPOCH, - - # Randomness and committees - latest_randao_mixes=Vector([ZERO_HASH for _ in range(LATEST_RANDAO_MIXES_LENGTH)]), - latest_start_shard=GENESIS_START_SHARD, - - # Finality - previous_epoch_attestations=[], - current_epoch_attestations=[], - previous_justified_epoch=GENESIS_EPOCH - 1, - current_justified_epoch=GENESIS_EPOCH, - previous_justified_root=ZERO_HASH, - current_justified_root=ZERO_HASH, - justification_bitfield=0, - finalized_epoch=GENESIS_EPOCH, - finalized_root=ZERO_HASH, - - # Recent state - latest_crosslinks=Vector([Crosslink(epoch=GENESIS_EPOCH, crosslink_data_root=ZERO_HASH) for _ in range(SHARD_COUNT)]), - latest_block_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), - latest_state_roots=Vector([ZERO_HASH for _ in range(SLOTS_PER_HISTORICAL_ROOT)]), - latest_active_index_roots=Vector([ZERO_HASH for _ in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH)]), - latest_slashed_balances=Vector([0 for _ in range(LATEST_SLASHED_EXIT_LENGTH)]), - latest_block_header=get_temporary_block_header(get_empty_block()), - historical_roots=[], - - # Ethereum 1.0 chain data - latest_eth1_data=genesis_eth1_data, - eth1_data_votes=[], - deposit_index=0, - ) - - # Process genesis deposits - for deposit in genesis_validator_deposits: - process_deposit(state, deposit) - - # Process genesis activations - for validator_index, _ in enumerate(state.validator_registry): - if get_effective_balance(state, validator_index) >= MAX_DEPOSIT_AMOUNT: - activate_validator(state, validator_index, is_genesis=True) - - genesis_active_index_root = hash_tree_root(get_active_validator_indices(state.validator_registry, GENESIS_EPOCH)) - for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): - state.latest_active_index_roots[index] = genesis_active_index_root - - return state - - -def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: - """ - Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found. - """ - if block.slot == slot: - return block - elif block.slot < slot: - return None - else: - return get_ancestor(store, store.get_parent(block), slot) - - -def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock: - """ - Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``. - """ - validators = start_state.validator_registry - active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot)) - attestation_targets = [ - (validator_index, get_latest_attestation_target(store, validator_index)) - for validator_index in active_validator_indices - ] - - # Use the rounded-balance-with-hysteresis supplied by the protocol for fork - # choice voting. This reduces the number of recomputations that need to be - # made for optimized implementations that precompute and save data - def get_vote_count(block: BeaconBlock) -> int: - return sum( - start_state.validator_registry[validator_index].high_balance - for validator_index, target in attestation_targets - if get_ancestor(store, target, block.slot) == block - ) - - head = start_block - while 1: - children = get_children(store, head) - if len(children) == 0: - return head - head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) - - -def cache_state(state: BeaconState) -> None: - previous_slot_state_root = hash_tree_root(state) - - # store the previous slot's post state transition root - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_slot_state_root - - # cache state root in stored latest_block_header if empty - if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = previous_slot_state_root - - # store latest known block for previous slot - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = signed_root(state.latest_block_header) - - -def get_current_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_current_epoch(state))) - - -def get_previous_total_balance(state: BeaconState) -> Gwei: - return get_total_balance(state, get_active_validator_indices(state.validator_registry, get_previous_epoch(state))) - - -def get_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: - output = set() - for a in attestations: - output = output.union(get_attestation_participants(state, a.data, a.aggregation_bitfield)) - return sorted(list(output)) - - -def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei: - return get_total_balance(state, get_attesting_indices(state, attestations)) - - -def get_current_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.current_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_current_epoch(state))) - ] - - -def get_previous_epoch_boundary_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations - if a.data.target_root == get_block_root(state, get_epoch_start_slot(get_previous_epoch(state))) - ] - - -def get_previous_epoch_matching_head_attestations(state: BeaconState) -> List[PendingAttestation]: - return [ - a for a in state.previous_epoch_attestations - if a.data.beacon_block_root == get_block_root(state, a.data.slot) - ] - - -def get_winning_root_and_participants(state: BeaconState, shard: Shard) -> Tuple[Bytes32, List[ValidatorIndex]]: - all_attestations = state.current_epoch_attestations + state.previous_epoch_attestations - valid_attestations = [ - a for a in all_attestations if a.data.previous_crosslink == state.latest_crosslinks[shard] - ] - all_roots = [a.data.crosslink_data_root for a in valid_attestations] - - # handle when no attestations for shard available - if len(all_roots) == 0: - return ZERO_HASH, [] - - def get_attestations_for(root) -> List[PendingAttestation]: - return [a for a in valid_attestations if a.data.crosslink_data_root == root] - - # Winning crosslink root is the root with the most votes for it, ties broken in favor of - # lexicographically higher hash - winning_root = max(all_roots, key=lambda r: (get_attesting_balance(state, get_attestations_for(r)), r)) - - return winning_root, get_attesting_indices(state, get_attestations_for(winning_root)) - - -def earliest_attestation(state: BeaconState, validator_index: ValidatorIndex) -> PendingAttestation: - return min([ - a for a in state.previous_epoch_attestations if - validator_index in get_attestation_participants(state, a.data, a.aggregation_bitfield) - ], key=lambda a: a.inclusion_slot) - - -def inclusion_slot(state: BeaconState, validator_index: ValidatorIndex) -> Slot: - return earliest_attestation(state, validator_index).inclusion_slot - - -def inclusion_distance(state: BeaconState, validator_index: ValidatorIndex) -> int: - attestation = earliest_attestation(state, validator_index) - return attestation.inclusion_slot - attestation.data.slot - - -def update_justification_and_finalization(state: BeaconState) -> None: - new_justified_epoch = state.current_justified_epoch - new_finalized_epoch = state.finalized_epoch - - # Rotate the justification bitfield up one epoch to make room for the current epoch - state.justification_bitfield <<= 1 - # If the previous epoch gets justified, fill the second last bit - previous_boundary_attesting_balance = get_attesting_balance(state, get_previous_epoch_boundary_attestations(state)) - if previous_boundary_attesting_balance * 3 >= get_previous_total_balance(state) * 2: - new_justified_epoch = get_current_epoch(state) - 1 - state.justification_bitfield |= 2 - # If the current epoch gets justified, fill the last bit - current_boundary_attesting_balance = get_attesting_balance(state, get_current_epoch_boundary_attestations(state)) - if current_boundary_attesting_balance * 3 >= get_current_total_balance(state) * 2: - new_justified_epoch = get_current_epoch(state) - state.justification_bitfield |= 1 - - # Process finalizations - bitfield = state.justification_bitfield - current_epoch = get_current_epoch(state) - # The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and state.previous_justified_epoch == current_epoch - 3: - new_finalized_epoch = state.previous_justified_epoch - # The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and state.previous_justified_epoch == current_epoch - 2: - new_finalized_epoch = state.previous_justified_epoch - # The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and state.current_justified_epoch == current_epoch - 2: - new_finalized_epoch = state.current_justified_epoch - # The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and state.current_justified_epoch == current_epoch - 1: - new_finalized_epoch = state.current_justified_epoch - - # Update state jusification/finality fields - state.previous_justified_epoch = state.current_justified_epoch - state.previous_justified_root = state.current_justified_root - if new_justified_epoch != state.current_justified_epoch: - state.current_justified_epoch = new_justified_epoch - state.current_justified_root = get_block_root(state, get_epoch_start_slot(new_justified_epoch)) - if new_finalized_epoch != state.finalized_epoch: - state.finalized_epoch = new_finalized_epoch - state.finalized_root = get_block_root(state, get_epoch_start_slot(new_finalized_epoch)) - - -def process_crosslinks(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - previous_epoch = max(current_epoch - 1, GENESIS_EPOCH) - next_epoch = current_epoch + 1 - for slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch)): - for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_root, participants = get_winning_root_and_participants(state, shard) - participating_balance = get_total_balance(state, participants) - total_balance = get_total_balance(state, crosslink_committee) - if 3 * participating_balance >= 2 * total_balance: - state.latest_crosslinks[shard] = Crosslink( - epoch=min(slot_to_epoch(slot), state.latest_crosslinks[shard].epoch + MAX_CROSSLINK_EPOCHS), - crosslink_data_root=winning_root - ) - - -def maybe_reset_eth1_period(state: BeaconState) -> None: - if (get_current_epoch(state) + 1) % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: - for eth1_data_vote in state.eth1_data_votes: - # If a majority of all votes were for a particular eth1_data value, - # then set that as the new canonical value - if eth1_data_vote.vote_count * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: - state.latest_eth1_data = eth1_data_vote.eth1_data - state.eth1_data_votes = [] - - -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - if get_previous_total_balance(state) == 0: - return 0 - - adjusted_quotient = integer_squareroot(get_previous_total_balance(state)) // BASE_REWARD_QUOTIENT - return get_effective_balance(state, index) // adjusted_quotient // 5 - - -def get_inactivity_penalty(state: BeaconState, index: ValidatorIndex, epochs_since_finality: int) -> Gwei: - if epochs_since_finality <= 4: - extra_penalty = 0 - else: - extra_penalty = get_effective_balance(state, index) * epochs_since_finality // INACTIVITY_PENALTY_QUOTIENT // 2 - return get_base_reward(state, index) + extra_penalty - - -def get_justification_and_finalization_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - current_epoch = get_current_epoch(state) - epochs_since_finality = current_epoch + 1 - state.finalized_epoch - rewards = [0 for index in range(len(state.validator_registry))] - penalties = [0 for index in range(len(state.validator_registry))] - # Some helper variables - boundary_attestations = get_previous_epoch_boundary_attestations(state) - boundary_attesting_balance = get_attesting_balance(state, boundary_attestations) - total_balance = get_previous_total_balance(state) - total_attesting_balance = get_attesting_balance(state, state.previous_epoch_attestations) - matching_head_attestations = get_previous_epoch_matching_head_attestations(state) - matching_head_balance = get_attesting_balance(state, matching_head_attestations) - eligible_validators = [ - index for index, validator in enumerate(state.validator_registry) - if ( - is_active_validator(validator, current_epoch) or - (validator.slashed and current_epoch < validator.withdrawable_epoch) - ) - ] - # Process rewards or penalties for all validators - for index in eligible_validators: - base_reward = get_base_reward(state, index) - # Expected FFG source - if index in get_attesting_indices(state, state.previous_epoch_attestations): - rewards[index] += base_reward * total_attesting_balance // total_balance - # Inclusion speed bonus - rewards[index] += ( - base_reward * MIN_ATTESTATION_INCLUSION_DELAY // - inclusion_distance(state, index) - ) - else: - penalties[index] += base_reward - # Expected FFG target - if index in get_attesting_indices(state, boundary_attestations): - rewards[index] += base_reward * boundary_attesting_balance // total_balance - else: - penalties[index] += get_inactivity_penalty(state, index, epochs_since_finality) - # Expected head - if index in get_attesting_indices(state, matching_head_attestations): - rewards[index] += base_reward * matching_head_balance // total_balance - else: - penalties[index] += base_reward - # Proposer bonus - if index in get_attesting_indices(state, state.previous_epoch_attestations): - proposer_index = get_beacon_proposer_index(state, inclusion_slot(state, index)) - rewards[proposer_index] += base_reward // ATTESTATION_INCLUSION_REWARD_QUOTIENT - # Take away max rewards if we're not finalizing - if epochs_since_finality > 4: - penalties[index] += base_reward * 4 - return [rewards, penalties] - - -def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: - rewards = [0 for index in range(len(state.validator_registry))] - penalties = [0 for index in range(len(state.validator_registry))] - previous_epoch_start_slot = get_epoch_start_slot(get_previous_epoch(state)) - current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) - for slot in range(previous_epoch_start_slot, current_epoch_start_slot): - for crosslink_committee, shard in get_crosslink_committees_at_slot(state, slot): - winning_root, participants = get_winning_root_and_participants(state, shard) - participating_balance = get_total_balance(state, participants) - total_balance = get_total_balance(state, crosslink_committee) - for index in crosslink_committee: - if index in participants: - rewards[index] += get_base_reward(state, index) * participating_balance // total_balance - else: - penalties[index] += get_base_reward(state, index) - return [rewards, penalties] - - -def apply_rewards(state: BeaconState) -> None: - rewards1, penalties1 = get_justification_and_finalization_deltas(state) - rewards2, penalties2 = get_crosslink_deltas(state) - for i in range(len(state.validator_registry)): - set_balance( - state, - i, - max( - 0, - get_balance(state, i) + rewards1[i] + rewards2[i] - penalties1[i] - penalties2[i], - ), - ) - - -def process_ejections(state: BeaconState) -> None: - """ - Iterate through the validator registry - and eject active validators with balance below ``EJECTION_BALANCE``. - """ - for index in get_active_validator_indices(state.validator_registry, get_current_epoch(state)): - if get_balance(state, index) < EJECTION_BALANCE: - initiate_validator_exit(state, index) - - -def update_validator_registry(state: BeaconState) -> None: - """ - Update validator registry. - Note that this function mutates ``state``. - """ - current_epoch = get_current_epoch(state) - # The active validators - active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) - # The total effective balance of active validators - total_balance = get_total_balance(state, active_validator_indices) - - # The maximum balance churn in Gwei (for deposits and exits separately) - max_balance_churn = max( - MAX_DEPOSIT_AMOUNT, - total_balance // (2 * MAX_BALANCE_CHURN_QUOTIENT) - ) - - # Activate validators within the allowable balance churn - balance_churn = 0 - for index, validator in enumerate(state.validator_registry): - if validator.activation_epoch == FAR_FUTURE_EPOCH and get_balance(state, index) >= MAX_DEPOSIT_AMOUNT: - # Check the balance churn would be within the allowance - balance_churn += get_effective_balance(state, index) - if balance_churn > max_balance_churn: - break - - # Activate validator - activate_validator(state, index, is_genesis=False) - - # Exit validators within the allowable balance churn - if current_epoch < state.validator_registry_update_epoch + LATEST_SLASHED_EXIT_LENGTH: - balance_churn = ( - state.latest_slashed_balances[state.validator_registry_update_epoch % LATEST_SLASHED_EXIT_LENGTH] - - state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - ) - - for index, validator in enumerate(state.validator_registry): - if validator.exit_epoch == FAR_FUTURE_EPOCH and validator.initiated_exit: - # Check the balance churn would be within the allowance - balance_churn += get_effective_balance(state, index) - if balance_churn > max_balance_churn: - break - - # Exit validator - exit_validator(state, index) - - state.validator_registry_update_epoch = current_epoch - - -def update_registry(state: BeaconState) -> None: - # Check if we should update, and if so, update - if state.finalized_epoch > state.validator_registry_update_epoch: - update_validator_registry(state) - state.latest_start_shard = ( - state.latest_start_shard + - get_current_epoch_committee_count(state) - ) % SHARD_COUNT - - -def process_slashings(state: BeaconState) -> None: - """ - Process the slashings. - Note that this function mutates ``state``. - """ - current_epoch = get_current_epoch(state) - active_validator_indices = get_active_validator_indices(state.validator_registry, current_epoch) - total_balance = get_total_balance(state, active_validator_indices) - - # Compute `total_penalties` - total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] - total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - total_penalties = total_at_end - total_at_start - - for index, validator in enumerate(state.validator_registry): - if validator.slashed and current_epoch == validator.withdrawable_epoch - LATEST_SLASHED_EXIT_LENGTH // 2: - penalty = max( - get_effective_balance(state, index) * min(total_penalties * 3, total_balance) // total_balance, - get_effective_balance(state, index) // MIN_PENALTY_QUOTIENT - ) - decrease_balance(state, index, penalty) - - -def process_exit_queue(state: BeaconState) -> None: - """ - Process the exit queue. - Note that this function mutates ``state``. - """ - def eligible(index): - validator = state.validator_registry[index] - # Filter out dequeued validators - if validator.withdrawable_epoch != FAR_FUTURE_EPOCH: - return False - # Dequeue if the minimum amount of time has passed - else: - return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY - - eligible_indices = filter(eligible, list(range(len(state.validator_registry)))) - # Sort in order of exit epoch, and validators that exit within the same epoch exit in order of validator index - sorted_indices = sorted(eligible_indices, key=lambda index: state.validator_registry[index].exit_epoch) - for dequeues, index in enumerate(sorted_indices): - if dequeues >= MAX_EXIT_DEQUEUES_PER_EPOCH: - break - prepare_validator_for_withdrawal(state, index) - - -def finish_epoch_update(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = current_epoch + 1 - # Set active index root - index_root_position = (next_epoch + ACTIVATION_EXIT_DELAY) % LATEST_ACTIVE_INDEX_ROOTS_LENGTH - state.latest_active_index_roots[index_root_position] = hash_tree_root( - get_active_validator_indices(state.validator_registry, next_epoch + ACTIVATION_EXIT_DELAY) - ) - # Set total slashed balances - state.latest_slashed_balances[next_epoch % LATEST_SLASHED_EXIT_LENGTH] = ( - state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] - ) - # Set randao mix - state.latest_randao_mixes[next_epoch % LATEST_RANDAO_MIXES_LENGTH] = get_randao_mix(state, current_epoch) - # Set historical root accumulator - if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: - historical_batch = HistoricalBatch( - block_roots=state.latest_block_roots, - state_roots=state.latest_state_roots, - ) - state.historical_roots.append(hash_tree_root(historical_batch)) - # Rotate current/previous epoch attestations - state.previous_epoch_attestations = state.current_epoch_attestations - state.current_epoch_attestations = [] - - -def advance_slot(state: BeaconState) -> None: - state.slot += 1 - - -def process_block_header(state: BeaconState, block: BeaconBlock) -> None: - # Verify that the slots match - assert block.slot == state.slot - # Verify that the parent matches - assert block.previous_block_root == signed_root(state.latest_block_header) - # Save current block as the new latest block - state.latest_block_header = get_temporary_block_header(block) - # Verify proposer is not slashed - proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] - assert not proposer.slashed - # Verify proposer signature - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(block), - signature=block.signature, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_BEACON_BLOCK) - ) - - -def process_randao(state: BeaconState, block: BeaconBlock) -> None: - proposer = state.validator_registry[get_beacon_proposer_index(state, state.slot)] - # Verify that the provided randao value is valid - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=hash_tree_root(get_current_epoch(state)), - signature=block.body.randao_reveal, - domain=get_domain(state.fork, get_current_epoch(state), DOMAIN_RANDAO) - ) - # Mix it in - state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( - xor(get_randao_mix(state, get_current_epoch(state)), - hash(block.body.randao_reveal)) - ) - - -def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: - for eth1_data_vote in state.eth1_data_votes: - # If someone else has already voted for the same hash, add to its counter - if eth1_data_vote.eth1_data == block.body.eth1_data: - eth1_data_vote.vote_count += 1 - return - # If we're seeing this hash for the first time, make a new counter - state.eth1_data_votes.append(Eth1DataVote(eth1_data=block.body.eth1_data, vote_count=1)) - - -def process_proposer_slashing(state: BeaconState, - proposer_slashing: ProposerSlashing) -> None: - """ - Process ``ProposerSlashing`` transaction. - Note that this function mutates ``state``. - """ - proposer = state.validator_registry[proposer_slashing.proposer_index] - # Verify that the epoch is the same - assert slot_to_epoch(proposer_slashing.header_1.slot) == slot_to_epoch(proposer_slashing.header_2.slot) - # But the headers are different - assert proposer_slashing.header_1 != proposer_slashing.header_2 - # Check proposer is slashable - assert is_slashable_validator(proposer, get_current_epoch(state)) - # Signatures are valid - for header in (proposer_slashing.header_1, proposer_slashing.header_2): - assert bls_verify( - pubkey=proposer.pubkey, - message_hash=signed_root(header), - signature=header.signature, - domain=get_domain(state.fork, slot_to_epoch(header.slot), DOMAIN_BEACON_BLOCK) - ) - slash_validator(state, proposer_slashing.proposer_index) - - -def process_attester_slashing(state: BeaconState, - attester_slashing: AttesterSlashing) -> None: - """ - Process ``AttesterSlashing`` transaction. - Note that this function mutates ``state``. - """ - attestation1 = attester_slashing.slashable_attestation_1 - attestation2 = attester_slashing.slashable_attestation_2 - # Check that the attestations are conflicting - assert attestation1.data != attestation2.data - assert ( - is_double_vote(attestation1.data, attestation2.data) or - is_surround_vote(attestation1.data, attestation2.data) - ) - assert verify_slashable_attestation(state, attestation1) - assert verify_slashable_attestation(state, attestation2) - slashable_indices = [ - index for index in attestation1.validator_indices - if ( - index in attestation2.validator_indices and - is_slashable_validator(state.validator_registry[index], get_current_epoch(state)) - ) - ] - assert len(slashable_indices) >= 1 - for index in slashable_indices: - slash_validator(state, index) - - -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - """ - Process ``Attestation`` transaction. - Note that this function mutates ``state``. - """ - assert max(GENESIS_SLOT, state.slot - SLOTS_PER_EPOCH) <= attestation.data.slot - assert attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY - - # Check target epoch, source epoch, and source root - target_epoch = slot_to_epoch(attestation.data.slot) - assert (target_epoch, attestation.data.source_epoch, attestation.data.source_root) in { - (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root), - (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root), - } - - # Check crosslink data - assert attestation.data.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] - assert state.latest_crosslinks[attestation.data.shard] in { - attestation.data.previous_crosslink, # Case 1: latest crosslink matches previous crosslink - Crosslink( # Case 2: latest crosslink matches current crosslink - crosslink_data_root=attestation.data.crosslink_data_root, - epoch=min(slot_to_epoch(attestation.data.slot), - attestation.data.previous_crosslink.epoch + MAX_CROSSLINK_EPOCHS) - ), - } - - # Check custody bits [to be generalised in phase 1] - assert attestation.custody_bitfield == b'\x00' * len(attestation.custody_bitfield) - - # Check aggregate signature [to be generalised in phase 1] - participants = get_attestation_participants(state, attestation.data, attestation.aggregation_bitfield) - assert len(participants) != 0 - assert bls_verify( - pubkey=bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in participants]), - message_hash=hash_tree_root(AttestationDataAndCustodyBit(data=attestation.data, custody_bit=0b0)), - signature=attestation.aggregate_signature, - domain=get_domain(state.fork, target_epoch, DOMAIN_ATTESTATION), - ) - - # Cache pending attestation - pending_attestation = PendingAttestation( - data=attestation.data, - aggregation_bitfield=attestation.aggregation_bitfield, - custody_bitfield=attestation.custody_bitfield, - inclusion_slot=state.slot - ) - if target_epoch == get_current_epoch(state): - state.current_epoch_attestations.append(pending_attestation) - else: - state.previous_epoch_attestations.append(pending_attestation) - - -def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: - """ - Process ``VoluntaryExit`` transaction. - Note that this function mutates ``state``. - """ - validator = state.validator_registry[exit.validator_index] - # Verify the validator is active - assert is_active_validator(validator, get_current_epoch(state)) - # Verify the validator has not yet exited - assert validator.exit_epoch == FAR_FUTURE_EPOCH - # Verify the validator has not initiated an exit - assert validator.initiated_exit is False - # Exits must specify an epoch when they become valid; they are not valid before then - assert get_current_epoch(state) >= exit.epoch - # Verify the validator has been active long enough - assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD - # Verify signature - assert bls_verify( - pubkey=validator.pubkey, - message_hash=signed_root(exit), - signature=exit.signature, - domain=get_domain(state.fork, exit.epoch, DOMAIN_VOLUNTARY_EXIT) - ) - # Initiate exit - initiate_validator_exit(state, exit.validator_index) - - -def process_transfer(state: BeaconState, transfer: Transfer) -> None: - """ - Process ``Transfer`` transaction. - Note that this function mutates ``state``. - """ - # Verify the amount and fee aren't individually too big (for anti-overflow purposes) - assert get_balance(state, transfer.sender) >= max(transfer.amount, transfer.fee) - # Verify that we have enough ETH to send, and that after the transfer the balance will be either - # exactly zero or at least MIN_DEPOSIT_AMOUNT - assert ( - get_balance(state, transfer.sender) == transfer.amount + transfer.fee or - get_balance(state, transfer.sender) >= transfer.amount + transfer.fee + MIN_DEPOSIT_AMOUNT - ) - # A transfer is valid in only one slot - assert state.slot == transfer.slot - # Only withdrawn or not-yet-deposited accounts can transfer - assert ( - get_current_epoch(state) >= state.validator_registry[transfer.sender].withdrawable_epoch or - state.validator_registry[transfer.sender].activation_epoch == FAR_FUTURE_EPOCH - ) - # Verify that the pubkey is valid - assert ( - state.validator_registry[transfer.sender].withdrawal_credentials == - BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] - ) - # Verify that the signature is valid - assert bls_verify( - pubkey=transfer.pubkey, - message_hash=signed_root(transfer), - signature=transfer.signature, - domain=get_domain(state.fork, slot_to_epoch(transfer.slot), DOMAIN_TRANSFER) - ) - # Process the transfer - decrease_balance(state, transfer.sender, transfer.amount + transfer.fee) - increase_balance(state, transfer.recipient, transfer.amount) - increase_balance(state, get_beacon_proposer_index(state, state.slot), transfer.fee) - - -def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: - assert block.state_root == hash_tree_root(state) - -# Monkey patch validator get committee code -_compute_committee = compute_committee -committee_cache = {} - - -def compute_committee(validator_indices: List[ValidatorIndex], - seed: Bytes32, - index: int, - total_committees: int) -> List[ValidatorIndex]: - - param_hash = (hash_tree_root(validator_indices), seed, index, total_committees) - - if param_hash in committee_cache: - # print("Cache hit, epoch={0}".format(epoch)) - return committee_cache[param_hash] - else: - # print("Cache miss, epoch={0}".format(epoch)) - ret = _compute_committee(validator_indices, seed, index, total_committees) - committee_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} - - -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret - \ No newline at end of file diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 64ae482fd7..6d57e59164 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -3,7 +3,7 @@ setup( name='pyspec', version='1.0', - packages=['eth2'], + packages=['eth2.debug', 'eth2.phase0', 'eth2.utils'], install_requires=[ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", From 05970c93831ca3b0db08e265355335ec4791ae25 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 23:10:16 +0800 Subject: [PATCH 10/61] fix pyspec setup, and update usages: py_tests and in docs --- py_tests/README.md | 3 ++- py_tests/conftest.py | 0 .../block_processing/test_process_attestation.py | 2 +- .../block_processing/test_process_block_header.py | 2 +- .../phase0/block_processing/test_process_deposit.py | 2 +- .../test_process_proposer_slashing.py | 2 +- .../phase0/block_processing/test_voluntary_exit.py | 2 +- py_tests/requirements.txt | 2 +- test_generators/README.md | 11 ++++++++--- {py_tests => test_libs/pyspec/eth2}/__init__.py | 0 test_libs/pyspec/setup.py | 5 ++--- 11 files changed, 18 insertions(+), 13 deletions(-) delete mode 100644 py_tests/conftest.py rename {py_tests => test_libs/pyspec/eth2}/__init__.py (100%) diff --git a/py_tests/README.md b/py_tests/README.md index 659e737f14..86e2b541db 100644 --- a/py_tests/README.md +++ b/py_tests/README.md @@ -13,9 +13,10 @@ From within the py_tests folder: Install dependencies: ```bash python3 -m venv venv -. py_tests/venv/bin/activate +. venv/bin/activate pip3 install -r requirements.txt ``` +Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement. Run the tests: ``` diff --git a/py_tests/conftest.py b/py_tests/conftest.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/py_tests/phase0/block_processing/test_process_attestation.py b/py_tests/phase0/block_processing/test_process_attestation.py index f76ca77d8c..ba27540923 100644 --- a/py_tests/phase0/block_processing/test_process_attestation.py +++ b/py_tests/phase0/block_processing/test_process_attestation.py @@ -11,7 +11,7 @@ process_attestation, slot_to_epoch, ) -from ..helpers import ( +from phase0.helpers import ( build_empty_block_for_next_slot, get_valid_attestation, ) diff --git a/py_tests/phase0/block_processing/test_process_block_header.py b/py_tests/phase0/block_processing/test_process_block_header.py index ef902bb727..0c4a930c29 100644 --- a/py_tests/phase0/block_processing/test_process_block_header.py +++ b/py_tests/phase0/block_processing/test_process_block_header.py @@ -8,7 +8,7 @@ advance_slot, process_block_header, ) -from ..helpers import ( +from phase0.helpers import ( build_empty_block_for_next_slot, ) diff --git a/py_tests/phase0/block_processing/test_process_deposit.py b/py_tests/phase0/block_processing/test_process_deposit.py index 19467984e3..bda014665b 100644 --- a/py_tests/phase0/block_processing/test_process_deposit.py +++ b/py_tests/phase0/block_processing/test_process_deposit.py @@ -8,7 +8,7 @@ ZERO_HASH, process_deposit, ) -from ..helpers import ( +from phase0.helpers import ( build_deposit, privkeys, pubkeys, diff --git a/py_tests/phase0/block_processing/test_process_proposer_slashing.py b/py_tests/phase0/block_processing/test_process_proposer_slashing.py index 761624c256..b172cd9881 100644 --- a/py_tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/py_tests/phase0/block_processing/test_process_proposer_slashing.py @@ -7,7 +7,7 @@ get_current_epoch, process_proposer_slashing, ) -from ..helpers import ( +from phase0.helpers import ( get_valid_proposer_slashing, ) diff --git a/py_tests/phase0/block_processing/test_voluntary_exit.py b/py_tests/phase0/block_processing/test_voluntary_exit.py index 45a8af1bbc..32c7470240 100644 --- a/py_tests/phase0/block_processing/test_voluntary_exit.py +++ b/py_tests/phase0/block_processing/test_voluntary_exit.py @@ -8,7 +8,7 @@ get_current_epoch, process_voluntary_exit, ) -from ..helpers import ( +from phase0.helpers import ( build_voluntary_exit, pubkey_to_privkey, ) diff --git a/py_tests/requirements.txt b/py_tests/requirements.txt index d18b29127e..27b3f22d8d 100644 --- a/py_tests/requirements.txt +++ b/py_tests/requirements.txt @@ -4,4 +4,4 @@ oyaml==0.7 pycryptodome==3.7.3 py_ecc>=1.6.0 pytest>=3.6,<3.7 -../pyspec +../test_libs/pyspec diff --git a/test_generators/README.md b/test_generators/README.md index 61dc960637..155059fe0f 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -46,9 +46,9 @@ Simply open up the generator (not all at once) of choice in your favorite IDE/ed ```bash # Create a virtual environment (any venv/.venv/.venvs is git-ignored) -python3 -m venv .venv +python3 -m venv venv # Activate the venv, this is where dependencies are installed for the generator -. .venv/bin/activate +. venv/bin/activate ``` Now that you have a virtual environment, write your generator. @@ -57,8 +57,13 @@ It's recommended to extend the base-generator. Create a `requirements.txt` in the root of your generator directory: ``` eth-utils==1.4.1 -../test_libs/gen_helpers +../../test_libs/gen_helpers ``` +And optionally, to include pyspec, add: +``` +../../test_libs/pyspec +``` +Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement. Install all the necessary requirements (re-run when you add more): ```bash diff --git a/py_tests/__init__.py b/test_libs/pyspec/eth2/__init__.py similarity index 100% rename from py_tests/__init__.py rename to test_libs/pyspec/eth2/__init__.py diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 6d57e59164..b04847d374 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -1,9 +1,8 @@ -from distutils.core import setup +from setuptools import setup, find_packages setup( name='pyspec', - version='1.0', - packages=['eth2.debug', 'eth2.phase0', 'eth2.utils'], + packages=find_packages(), install_requires=[ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", From 58f09b20f5c9b8bf75b486b8aefaa5f7b5384919 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 23:23:36 +0800 Subject: [PATCH 11/61] update CI, makefile has test runner now --- .circleci/config.yml | 8 +++----- Makefile | 5 ++--- py_tests/README.md | 6 ++++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 411eb92308..efe4d07230 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,14 +8,12 @@ jobs: steps: - checkout - run: - name: Build phase0 spec - command: make phase0 + name: Build pyspec + command: make pyspec - run: name: run py-tests - command: | - . venv/bin/activate - pytest tests + command: make test - run: name: Generate YAML tests diff --git a/Makefile b/Makefile index 84dbd0c264..65839254a1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts TEST_LIBS_DIR = ./test_libs PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec - +PY_TEST_DIR = ./py_tests YAML_TEST_DIR = ./yaml_tests GENERATOR_DIR = ./test_generators GENERATOR_VENVS_DIR = $(GENERATOR_DIR)/.venvs @@ -29,9 +29,8 @@ clean: yaml_tests: $(YAML_TEST_TARGETS) # runs a limited set of tests against a minimal config -# run pytest with `-m` option to full suite test: $(PY_SPEC_TARGETS) - pytest -m minimal_config tests/ + cd $(PY_TEST_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; pytest -m minimal_config . # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_TARGETS) diff --git a/py_tests/README.md b/py_tests/README.md index 86e2b541db..ca2bed4ccd 100644 --- a/py_tests/README.md +++ b/py_tests/README.md @@ -8,6 +8,12 @@ There are ideas to port these tests to the YAML test suite, ## How to run tests +### Automated + +Run `make test` from the root of the spec repository. + +### Manual + From within the py_tests folder: Install dependencies: From f2703bc8d3bd43b1dcfeafbb49b462d499409809 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 28 Mar 2019 23:27:28 +0800 Subject: [PATCH 12/61] update readme with example pyspec import --- test_generators/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_generators/README.md b/test_generators/README.md index 155059fe0f..e72f9caf41 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -109,6 +109,12 @@ if __name__ == "__main__": ``` +And to use the pyspec: + +``` +from eth2.phase0 import spec +``` + Recommendations: - you can have more than just 1 generator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` - you can concatenate lists of test cases, if you don't want to split it up in suites. From adf91f50c579a0a603add2270a956b3f543b500c Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 29 Mar 2019 00:05:40 +0800 Subject: [PATCH 13/61] run yaml test generators from makefile --- Makefile | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 65839254a1..566db0a7ac 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,12 @@ PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec PY_TEST_DIR = ./py_tests YAML_TEST_DIR = ./yaml_tests GENERATOR_DIR = ./test_generators -GENERATOR_VENVS_DIR = $(GENERATOR_DIR)/.venvs # Collect a list of generator names GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) # Map this list of generator paths to a list of test output paths YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS)) +GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%/venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2/phase0/spec.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -22,11 +22,12 @@ all: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) $(PY_SPEC_ALL_TARGETS) clean: rm -rf $(YAML_TEST_DIR) - rm -rf $(GENERATOR_VENVS_DIR) + rm -rf $(GENERATOR_VENVS) + rm -rf $(PY_TEST_DIR)/venv rm -rf $(PY_SPEC_ALL_TARGETS) -# "make yaml_tests" to run generators -yaml_tests: $(YAML_TEST_TARGETS) +# "make gen_yaml_tests" to run generators +gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) # runs a limited set of tests against a minimal config test: $(PY_SPEC_TARGETS) @@ -43,6 +44,7 @@ $(PY_SPEC_DIR)/eth2/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ +CURRENT_DIR = ${CURDIR} # The function that builds a set of suite files, by calling a generator for the given type (param 1) define build_yaml_tests @@ -50,22 +52,19 @@ define build_yaml_tests # Create the output mkdir -p $(YAML_TEST_DIR)$(1) - # Create a virtual environment - python3 -m venv $(VENV_DIR)$(1) - # Activate the venv, this is where dependencies are installed for the generator - . $(GENERATOR_VENVS_DIR)$(1)bin/activate - # Install all the necessary requirements - pip3 install -r $(GENERATOR_DIR)$(1)requirements.txt + # 1) Create a virtual environment + # 2) Activate the venv, this is where dependencies are installed for the generator + # 3) Install all the necessary requirements + # 4) Run the generator. The generator is assumed to have an "main.py" file. + # 5) We output to the tests dir (generator program should accept a "-o " argument. + cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) - # Run the generator. The generator is assumed to have an "main.py" file. - # We output to the tests dir (generator program should accept a "-p " argument. - python3 $(GENERATOR_DIR)$(1)main.py -o $(YAML_TEST_DIR)$(1) $(info generator $(1) finished) endef # The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary) $(YAML_TEST_DIR): - $(info ${YAML_TEST_TARGETS}) + $(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS}) mkdir -p $@ # For any target within the tests dir, build it using the build_yaml_tests function. From 04d41ddabeb5e7d29b0ab38c75ce0f7331d6ec1d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 29 Mar 2019 00:16:18 +0800 Subject: [PATCH 14/61] small fixes/tweaks, pytests and yaml generators work well now --- .circleci/config.yml | 4 ++-- Makefile | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index efe4d07230..c347a064f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,12 +12,12 @@ jobs: command: make pyspec - run: - name: run py-tests + name: Run py-tests command: make test - run: name: Generate YAML tests - command: make yaml_tests + command: make gen_yaml_tests - store_artifacts: path: test-reports diff --git a/Makefile b/Makefile index 566db0a7ac..ca7321c223 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,13 @@ GENERATOR_DIR = ./test_generators GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) # Map this list of generator paths to a list of test output paths YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS)) -GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%/venv, $(GENERATORS)) +GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2/phase0/spec.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) -.PHONY: clean all test yaml_tests pyspec phase0 +.PHONY: clean all test gen_yaml_tests pyspec phase0 all: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) $(PY_SPEC_ALL_TARGETS) @@ -30,11 +30,11 @@ clean: gen_yaml_tests: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) # runs a limited set of tests against a minimal config -test: $(PY_SPEC_TARGETS) +test: $(PY_SPEC_ALL_TARGETS) cd $(PY_TEST_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; pytest -m minimal_config . # "make pyspec" to create the pyspec for all phases. -pyspec: $(PY_SPEC_TARGETS) +pyspec: $(PY_SPEC_ALL_TARGETS) # "make phase0" to create pyspec for phase0 phase0: $(PY_SPEC_PHASE_0_TARGETS) From 3b132b71aabdaf695eedbaee706ed7d9e3d5d019 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 29 Mar 2019 00:24:18 +0800 Subject: [PATCH 15/61] refine makefile, update generator doc --- Makefile | 3 ++- test_generators/README.md | 18 +++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index ca7321c223..93c2e49fa0 100644 --- a/Makefile +++ b/Makefile @@ -68,5 +68,6 @@ $(YAML_TEST_DIR): mkdir -p $@ # For any target within the tests dir, build it using the build_yaml_tests function. -$(YAML_TEST_DIR)%: +# (creation of output dir is a dependency) +$(YAML_TEST_DIR)%: $(YAML_TEST_DIR) $(call build_yaml_tests,$*) diff --git a/test_generators/README.md b/test_generators/README.md index e72f9caf41..f4dca7977e 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -17,7 +17,7 @@ pre-requisites: ### Cleaning -This removes the existing virtual environments (`/test_generators/.venvs/`), and generated tests (`/yaml_tests/`). +This removes the existing virtual environments (`/test_generators//venv`), and generated tests (`/yaml_tests/`). ```bash make clean @@ -28,7 +28,7 @@ make clean This runs all the generators. ```bash -make all +make gen_yaml_tests ``` ### Running a single generator @@ -37,7 +37,7 @@ The make file auto-detects generators in the `test_generators/` directory, and provides a tests-gen target for each generator, see example. ```bash -make ./tests/shuffling/ +make ./yaml_tests/shuffling/ ``` ## Developing a generator @@ -45,6 +45,7 @@ make ./tests/shuffling/ Simply open up the generator (not all at once) of choice in your favorite IDE/editor, and run: ```bash +# From the root of the generator directory: # Create a virtual environment (any venv/.venv/.venvs is git-ignored) python3 -m venv venv # Activate the venv, this is where dependencies are installed for the generator @@ -143,20 +144,11 @@ Note: you do not have to change the makefile. However, if necessary (e.g. not using python, or mixing in other languages), submit an issue, and it can be a special case. Do note that generators should be easy to maintain, lean, and based on the spec. -All of this should be done in a pull request to the master branch. - -To deploy new tests to the testing repository: - -1. Create a release tag with a new version number on Github. -2. Increment either the: - - major version, to indicate a change in the general testing format - - minor version, if a new test generator has been added - - path version, in other cases. ## How to remove a test generator If a test generator is not needed anymore, undo the steps described above and make a new release: -1. remove the generator folder +1. remove the generator directory 2. remove the generated tests in the `eth2.0-tests` repository by opening a PR there. 3. make a new release From ec37645e8f16114b03700438511934f7b692da6f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 29 Mar 2019 00:51:49 +0800 Subject: [PATCH 16/61] update pyspec readme --- test_libs/pyspec/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 25ee737f79..ee6bc5325f 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -7,9 +7,19 @@ With this executable spec, test-generators can easily create test-vectors for client implementations, and the spec itself can be verified to be consistent and coherent, through sanity tests implemented with pytest. +## Building + +All the dynamic parts of the spec can be build at once with `make pyspec`. + +Alternatively, you can build a sub-set of the pyspec: `make phase0`. + +Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2/phase0/spec.py` + +## Contributing + Contributions are welcome, but consider implementing your idea as part of the spec itself first. The pyspec is not a replacement. -If you see opportunity to include any of the `utils/` code in the spec, +If you see opportunity to include any of the `eth2/utils/` code in the spec, please submit an issue or PR. ## License From 00c3c1e2a6a121d77efdab576e9bdbcccae4e5a0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 30 Mar 2019 00:24:04 +0800 Subject: [PATCH 17/61] rename eth2 pkg to pyspec, per request of hww --- .gitignore | 2 +- Makefile | 8 ++++---- .../block_processing/test_process_attestation.py | 6 +++--- .../block_processing/test_process_block_header.py | 2 +- .../phase0/block_processing/test_process_deposit.py | 4 ++-- .../block_processing/test_process_proposer_slashing.py | 4 ++-- .../phase0/block_processing/test_voluntary_exit.py | 4 ++-- py_tests/phase0/conftest.py | 2 +- py_tests/phase0/helpers.py | 8 ++++---- py_tests/phase0/test_sanity.py | 10 +++++----- scripts/phase0/build_spec.py | 4 ++-- test_generators/README.md | 2 +- test_libs/pyspec/README.md | 4 ++-- test_libs/pyspec/{eth2 => pyspec}/__init__.py | 0 test_libs/pyspec/{eth2 => pyspec}/debug/__init__.py | 0 test_libs/pyspec/{eth2 => pyspec}/debug/jsonize.py | 2 +- test_libs/pyspec/{eth2 => pyspec}/phase0/__init__.py | 0 .../pyspec/{eth2 => pyspec}/phase0/state_transition.py | 0 test_libs/pyspec/{eth2 => pyspec}/utils/__init__.py | 0 test_libs/pyspec/{eth2 => pyspec}/utils/bls_stub.py | 0 .../pyspec/{eth2 => pyspec}/utils/hash_function.py | 0 .../pyspec/{eth2 => pyspec}/utils/merkle_minimal.py | 0 test_libs/pyspec/{eth2 => pyspec}/utils/minimal_ssz.py | 0 23 files changed, 31 insertions(+), 31 deletions(-) rename test_libs/pyspec/{eth2 => pyspec}/__init__.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/debug/__init__.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/debug/jsonize.py (97%) rename test_libs/pyspec/{eth2 => pyspec}/phase0/__init__.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/phase0/state_transition.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/utils/__init__.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/utils/bls_stub.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/utils/hash_function.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/utils/merkle_minimal.py (100%) rename test_libs/pyspec/{eth2 => pyspec}/utils/minimal_ssz.py (100%) diff --git a/.gitignore b/.gitignore index b12e536ffe..909996e737 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ yaml_tests/ .pytest_cache # Dynamically built from Markdown spec -test_libs/pyspec/eth2/phase0/spec.py +test_libs/pyspec/pyspec/phase0/spec.py diff --git a/Makefile b/Makefile index 93c2e49fa0..34d3471183 100644 --- a/Makefile +++ b/Makefile @@ -12,18 +12,18 @@ GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS)) GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) -PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2/phase0/spec.py +PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/pyspec/phase0/spec.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) .PHONY: clean all test gen_yaml_tests pyspec phase0 -all: $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) $(PY_SPEC_ALL_TARGETS) +all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) clean: rm -rf $(YAML_TEST_DIR) rm -rf $(GENERATOR_VENVS) - rm -rf $(PY_TEST_DIR)/venv + rm -rf $(PY_TEST_DIR)/venv $(PY_TEST_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) # "make gen_yaml_tests" to run generators @@ -40,7 +40,7 @@ pyspec: $(PY_SPEC_ALL_TARGETS) phase0: $(PY_SPEC_PHASE_0_TARGETS) -$(PY_SPEC_DIR)/eth2/phase0/spec.py: +$(PY_SPEC_DIR)/pyspec/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ diff --git a/py_tests/phase0/block_processing/test_process_attestation.py b/py_tests/phase0/block_processing/test_process_attestation.py index ef6c0469bd..d454d6be45 100644 --- a/py_tests/phase0/block_processing/test_process_attestation.py +++ b/py_tests/phase0/block_processing/test_process_attestation.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2.phase0.spec as spec +import pyspec.phase0.spec as spec -from eth2.phase0.state_transition import ( +from pyspec.phase0.state_transition import ( state_transition, ) -from eth2.phase0.spec import ( +from pyspec.phase0.spec import ( get_current_epoch, process_attestation, slot_to_epoch, diff --git a/py_tests/phase0/block_processing/test_process_block_header.py b/py_tests/phase0/block_processing/test_process_block_header.py index 0c4a930c29..6c40260d56 100644 --- a/py_tests/phase0/block_processing/test_process_block_header.py +++ b/py_tests/phase0/block_processing/test_process_block_header.py @@ -2,7 +2,7 @@ import pytest -from eth2.phase0.spec import ( +from pyspec.phase0.spec import ( get_beacon_proposer_index, cache_state, advance_slot, diff --git a/py_tests/phase0/block_processing/test_process_deposit.py b/py_tests/phase0/block_processing/test_process_deposit.py index bda014665b..cf911e29ac 100644 --- a/py_tests/phase0/block_processing/test_process_deposit.py +++ b/py_tests/phase0/block_processing/test_process_deposit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2.phase0.spec as spec +import pyspec.phase0.spec as spec -from eth2.phase0.spec import ( +from pyspec.phase0.spec import ( get_balance, ZERO_HASH, process_deposit, diff --git a/py_tests/phase0/block_processing/test_process_proposer_slashing.py b/py_tests/phase0/block_processing/test_process_proposer_slashing.py index b172cd9881..3c3208b87e 100644 --- a/py_tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/py_tests/phase0/block_processing/test_process_proposer_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import eth2.phase0.spec as spec -from eth2.phase0.spec import ( +import pyspec.phase0.spec as spec +from pyspec.phase0.spec import ( get_balance, get_current_epoch, process_proposer_slashing, diff --git a/py_tests/phase0/block_processing/test_voluntary_exit.py b/py_tests/phase0/block_processing/test_voluntary_exit.py index 32c7470240..e7457126dd 100644 --- a/py_tests/phase0/block_processing/test_voluntary_exit.py +++ b/py_tests/phase0/block_processing/test_voluntary_exit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2.phase0.spec as spec +import pyspec.phase0.spec as spec -from eth2.phase0.spec import ( +from pyspec.phase0.spec import ( get_active_validator_indices, get_current_epoch, process_voluntary_exit, diff --git a/py_tests/phase0/conftest.py b/py_tests/phase0/conftest.py index 0def66ad6e..fb866160a7 100644 --- a/py_tests/phase0/conftest.py +++ b/py_tests/phase0/conftest.py @@ -1,6 +1,6 @@ import pytest -from eth2.phase0 import spec +from pyspec.phase0 import spec from .helpers import ( create_genesis_state, diff --git a/py_tests/phase0/helpers.py b/py_tests/phase0/helpers.py index 1cf7b284b4..5232023611 100644 --- a/py_tests/phase0/helpers.py +++ b/py_tests/phase0/helpers.py @@ -2,9 +2,9 @@ from py_ecc import bls -import eth2.phase0.spec as spec -from eth2.utils.minimal_ssz import signed_root -from eth2.phase0.spec import ( +import pyspec.phase0.spec as spec +from pyspec.utils.minimal_ssz import signed_root +from pyspec.phase0.spec import ( # constants EMPTY_SIGNATURE, ZERO_HASH, @@ -33,7 +33,7 @@ verify_merkle_branch, hash, ) -from eth2.utils.merkle_minimal import ( +from pyspec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/py_tests/phase0/test_sanity.py b/py_tests/phase0/test_sanity.py index 0af65ad51b..71f86d2a64 100644 --- a/py_tests/phase0/test_sanity.py +++ b/py_tests/phase0/test_sanity.py @@ -3,10 +3,10 @@ import pytest from py_ecc import bls -import eth2.phase0.spec as spec +import pyspec.phase0.spec as spec -from eth2.utils.minimal_ssz import signed_root -from eth2.phase0.spec import ( +from pyspec.utils.minimal_ssz import signed_root +from pyspec.phase0.spec import ( # constants EMPTY_SIGNATURE, ZERO_HASH, @@ -27,10 +27,10 @@ verify_merkle_branch, hash, ) -from eth2.phase0.state_transition import ( +from pyspec.phase0.state_transition import ( state_transition, ) -from eth2.utils.merkle_minimal import ( +from pyspec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 9f33c58834..8ec57d56d6 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -13,8 +13,8 @@ def build_spec(sourcefile, outfile): NewType, Tuple, ) -from eth2.utils.minimal_ssz import * -from eth2.utils.bls_stub import * +from pyspec.utils.minimal_ssz import * +from pyspec.utils.bls_stub import * """) diff --git a/test_generators/README.md b/test_generators/README.md index f4dca7977e..51cca65612 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -113,7 +113,7 @@ if __name__ == "__main__": And to use the pyspec: ``` -from eth2.phase0 import spec +from pyspec.phase0 import spec ``` Recommendations: diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index ee6bc5325f..11ffa835a7 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -13,13 +13,13 @@ All the dynamic parts of the spec can be build at once with `make pyspec`. Alternatively, you can build a sub-set of the pyspec: `make phase0`. -Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2/phase0/spec.py` +Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/pyspec/phase0/spec.py` ## Contributing Contributions are welcome, but consider implementing your idea as part of the spec itself first. The pyspec is not a replacement. -If you see opportunity to include any of the `eth2/utils/` code in the spec, +If you see opportunity to include any of the `pyspec/utils/` code in the spec, please submit an issue or PR. ## License diff --git a/test_libs/pyspec/eth2/__init__.py b/test_libs/pyspec/pyspec/__init__.py similarity index 100% rename from test_libs/pyspec/eth2/__init__.py rename to test_libs/pyspec/pyspec/__init__.py diff --git a/test_libs/pyspec/eth2/debug/__init__.py b/test_libs/pyspec/pyspec/debug/__init__.py similarity index 100% rename from test_libs/pyspec/eth2/debug/__init__.py rename to test_libs/pyspec/pyspec/debug/__init__.py diff --git a/test_libs/pyspec/eth2/debug/jsonize.py b/test_libs/pyspec/pyspec/debug/jsonize.py similarity index 97% rename from test_libs/pyspec/eth2/debug/jsonize.py rename to test_libs/pyspec/pyspec/debug/jsonize.py index 660e6b070d..a77684543a 100644 --- a/test_libs/pyspec/eth2/debug/jsonize.py +++ b/test_libs/pyspec/pyspec/debug/jsonize.py @@ -1,4 +1,4 @@ -from eth2.utils.minimal_ssz import hash_tree_root +from pyspec.utils.minimal_ssz import hash_tree_root def jsonize(value, typ, include_hash_tree_roots=False): diff --git a/test_libs/pyspec/eth2/phase0/__init__.py b/test_libs/pyspec/pyspec/phase0/__init__.py similarity index 100% rename from test_libs/pyspec/eth2/phase0/__init__.py rename to test_libs/pyspec/pyspec/phase0/__init__.py diff --git a/test_libs/pyspec/eth2/phase0/state_transition.py b/test_libs/pyspec/pyspec/phase0/state_transition.py similarity index 100% rename from test_libs/pyspec/eth2/phase0/state_transition.py rename to test_libs/pyspec/pyspec/phase0/state_transition.py diff --git a/test_libs/pyspec/eth2/utils/__init__.py b/test_libs/pyspec/pyspec/utils/__init__.py similarity index 100% rename from test_libs/pyspec/eth2/utils/__init__.py rename to test_libs/pyspec/pyspec/utils/__init__.py diff --git a/test_libs/pyspec/eth2/utils/bls_stub.py b/test_libs/pyspec/pyspec/utils/bls_stub.py similarity index 100% rename from test_libs/pyspec/eth2/utils/bls_stub.py rename to test_libs/pyspec/pyspec/utils/bls_stub.py diff --git a/test_libs/pyspec/eth2/utils/hash_function.py b/test_libs/pyspec/pyspec/utils/hash_function.py similarity index 100% rename from test_libs/pyspec/eth2/utils/hash_function.py rename to test_libs/pyspec/pyspec/utils/hash_function.py diff --git a/test_libs/pyspec/eth2/utils/merkle_minimal.py b/test_libs/pyspec/pyspec/utils/merkle_minimal.py similarity index 100% rename from test_libs/pyspec/eth2/utils/merkle_minimal.py rename to test_libs/pyspec/pyspec/utils/merkle_minimal.py diff --git a/test_libs/pyspec/eth2/utils/minimal_ssz.py b/test_libs/pyspec/pyspec/utils/minimal_ssz.py similarity index 100% rename from test_libs/pyspec/eth2/utils/minimal_ssz.py rename to test_libs/pyspec/pyspec/utils/minimal_ssz.py From f8cdd66ffcd81275d0cd3273b3e98924e0a93169 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 3 Apr 2019 13:50:56 +1100 Subject: [PATCH 18/61] minor fixes --- README.md | 1 + test_generators/bls/README.md | 3 ++- test_generators/bls/main.py | 6 +----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 294dd439d6..6ea9f1c14b 100644 --- a/README.md +++ b/README.md @@ -35,4 +35,5 @@ The following are the broad design goals for Ethereum 2.0: Documentation on the different components used during spec writing can be found here: * [YAML Test Generators](test_generators/README.md) * [Executable Python Spec](test_libs/pyspec/README.md) +* [Py-tests](py_tests/README.md) diff --git a/test_generators/bls/README.md b/test_generators/bls/README.md index 9ce1b2f6cc..a21ad16d94 100644 --- a/test_generators/bls/README.md +++ b/test_generators/bls/README.md @@ -17,4 +17,5 @@ The base unit is bytes48 of which only 381 bits are used ## Comments -Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set). \ No newline at end of file +Compared to Zcash, Ethereum specs always requires the compressed form (c_flag / most significant bit always set). +Also note that pubkeys and privkeys are reversed. diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 3c2ab454be..4c19c62491 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -1,7 +1,5 @@ """ BLS test vectors generator -Usage: - "python tgen_bls path/to/output.yml" """ # Standard library @@ -10,13 +8,11 @@ # Third-party import yaml +from py_ecc import bls # Ethereum from eth_utils import int_to_big_endian, big_endian_to_int -# Local imports -from py_ecc import bls - def int_to_hex(n: int) -> str: return '0x' + int_to_big_endian(n).hex() From 9e010da1187b982f84f102a0a198e2f199ef3775 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 3 Apr 2019 14:18:17 +1100 Subject: [PATCH 19/61] rename pyspec pkg to eth2spec --- .gitignore | 2 +- Makefile | 4 ++-- README.md | 2 +- .../block_processing/test_process_attestation.py | 6 +++--- .../block_processing/test_process_attester_slashing.py | 4 ++-- .../block_processing/test_process_block_header.py | 2 +- .../phase0/block_processing/test_process_deposit.py | 4 ++-- .../block_processing/test_process_proposer_slashing.py | 4 ++-- .../phase0/block_processing/test_voluntary_exit.py | 4 ++-- py_tests/phase0/conftest.py | 2 +- py_tests/phase0/helpers.py | 8 ++++---- py_tests/phase0/test_sanity.py | 10 +++++----- scripts/phase0/build_spec.py | 4 ++-- test_generators/README.md | 2 +- test_libs/pyspec/README.md | 4 ++-- test_libs/pyspec/{pyspec => eth2spec}/__init__.py | 0 .../pyspec/{pyspec => eth2spec}/debug/__init__.py | 0 test_libs/pyspec/{pyspec => eth2spec}/debug/jsonize.py | 2 +- .../pyspec/{pyspec => eth2spec}/phase0/__init__.py | 0 .../{pyspec => eth2spec}/phase0/state_transition.py | 0 .../pyspec/{pyspec => eth2spec}/utils/__init__.py | 0 .../pyspec/{pyspec => eth2spec}/utils/bls_stub.py | 0 .../pyspec/{pyspec => eth2spec}/utils/hash_function.py | 0 .../{pyspec => eth2spec}/utils/merkle_minimal.py | 0 .../pyspec/{pyspec => eth2spec}/utils/minimal_ssz.py | 0 25 files changed, 32 insertions(+), 32 deletions(-) rename test_libs/pyspec/{pyspec => eth2spec}/__init__.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/debug/__init__.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/debug/jsonize.py (97%) rename test_libs/pyspec/{pyspec => eth2spec}/phase0/__init__.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/phase0/state_transition.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/utils/__init__.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/utils/bls_stub.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/utils/hash_function.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/utils/merkle_minimal.py (100%) rename test_libs/pyspec/{pyspec => eth2spec}/utils/minimal_ssz.py (100%) diff --git a/.gitignore b/.gitignore index 909996e737..ce047240aa 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ yaml_tests/ .pytest_cache # Dynamically built from Markdown spec -test_libs/pyspec/pyspec/phase0/spec.py +test_libs/pyspec/eth2spec/phase0/spec.py diff --git a/Makefile b/Makefile index 34d3471183..bf23a14427 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS)) GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) -PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/pyspec/phase0/spec.py +PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -40,7 +40,7 @@ pyspec: $(PY_SPEC_ALL_TARGETS) phase0: $(PY_SPEC_PHASE_0_TARGETS) -$(PY_SPEC_DIR)/pyspec/phase0/spec.py: +$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ diff --git a/README.md b/README.md index d68c55cf37..e575aebd64 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,6 @@ The following are the broad design goals for Ethereum 2.0: Documentation on the different components used during spec writing can be found here: * [YAML Test Generators](test_generators/README.md) -* [Executable Python Spec](test_libs/pyspec/README.md) +* [Executable Python Spec](test_libs/eth2spec/README.md) * [Py-tests](py_tests/README.md) diff --git a/py_tests/phase0/block_processing/test_process_attestation.py b/py_tests/phase0/block_processing/test_process_attestation.py index d454d6be45..bd4c7da120 100644 --- a/py_tests/phase0/block_processing/test_process_attestation.py +++ b/py_tests/phase0/block_processing/test_process_attestation.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import pyspec.phase0.spec as spec +import eth2spec.phase0.spec as spec -from pyspec.phase0.state_transition import ( +from eth2spec.phase0.state_transition import ( state_transition, ) -from pyspec.phase0.spec import ( +from eth2spec.phase0.spec import ( get_current_epoch, process_attestation, slot_to_epoch, diff --git a/py_tests/phase0/block_processing/test_process_attester_slashing.py b/py_tests/phase0/block_processing/test_process_attester_slashing.py index b1f836a29d..5c97eb97b6 100644 --- a/py_tests/phase0/block_processing/test_process_attester_slashing.py +++ b/py_tests/phase0/block_processing/test_process_attester_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import pyspec.phase0.spec as spec -from pyspec.phase0.spec import ( +import eth2spec.phase0.spec as spec +from eth2spec.phase0.spec import ( get_balance, get_beacon_proposer_index, process_attester_slashing, diff --git a/py_tests/phase0/block_processing/test_process_block_header.py b/py_tests/phase0/block_processing/test_process_block_header.py index 6c40260d56..0ffc6f6133 100644 --- a/py_tests/phase0/block_processing/test_process_block_header.py +++ b/py_tests/phase0/block_processing/test_process_block_header.py @@ -2,7 +2,7 @@ import pytest -from pyspec.phase0.spec import ( +from eth2spec.phase0.spec import ( get_beacon_proposer_index, cache_state, advance_slot, diff --git a/py_tests/phase0/block_processing/test_process_deposit.py b/py_tests/phase0/block_processing/test_process_deposit.py index cf911e29ac..a424e28468 100644 --- a/py_tests/phase0/block_processing/test_process_deposit.py +++ b/py_tests/phase0/block_processing/test_process_deposit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import pyspec.phase0.spec as spec +import eth2spec.phase0.spec as spec -from pyspec.phase0.spec import ( +from eth2spec.phase0.spec import ( get_balance, ZERO_HASH, process_deposit, diff --git a/py_tests/phase0/block_processing/test_process_proposer_slashing.py b/py_tests/phase0/block_processing/test_process_proposer_slashing.py index 3c3208b87e..51e56bf70d 100644 --- a/py_tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/py_tests/phase0/block_processing/test_process_proposer_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import pyspec.phase0.spec as spec -from pyspec.phase0.spec import ( +import eth2spec.phase0.spec as spec +from eth2spec.phase0.spec import ( get_balance, get_current_epoch, process_proposer_slashing, diff --git a/py_tests/phase0/block_processing/test_voluntary_exit.py b/py_tests/phase0/block_processing/test_voluntary_exit.py index e7457126dd..51b44b5dc7 100644 --- a/py_tests/phase0/block_processing/test_voluntary_exit.py +++ b/py_tests/phase0/block_processing/test_voluntary_exit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import pyspec.phase0.spec as spec +import eth2spec.phase0.spec as spec -from pyspec.phase0.spec import ( +from eth2spec.phase0.spec import ( get_active_validator_indices, get_current_epoch, process_voluntary_exit, diff --git a/py_tests/phase0/conftest.py b/py_tests/phase0/conftest.py index fb866160a7..ec499862c8 100644 --- a/py_tests/phase0/conftest.py +++ b/py_tests/phase0/conftest.py @@ -1,6 +1,6 @@ import pytest -from pyspec.phase0 import spec +from eth2spec.phase0 import spec from .helpers import ( create_genesis_state, diff --git a/py_tests/phase0/helpers.py b/py_tests/phase0/helpers.py index 9ca05ba975..8ef6b53290 100644 --- a/py_tests/phase0/helpers.py +++ b/py_tests/phase0/helpers.py @@ -2,9 +2,9 @@ from py_ecc import bls -import pyspec.phase0.spec as spec -from pyspec.utils.minimal_ssz import signed_root -from pyspec.phase0.spec import ( +import eth2spec.phase0.spec as spec +from eth2spec.utils.minimal_ssz import signed_root +from eth2spec.phase0.spec import ( # constants EMPTY_SIGNATURE, ZERO_HASH, @@ -35,7 +35,7 @@ verify_merkle_branch, hash, ) -from pyspec.utils.merkle_minimal import ( +from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/py_tests/phase0/test_sanity.py b/py_tests/phase0/test_sanity.py index 3ca3905474..8c30b62f0a 100644 --- a/py_tests/phase0/test_sanity.py +++ b/py_tests/phase0/test_sanity.py @@ -3,10 +3,10 @@ import pytest from py_ecc import bls -import pyspec.phase0.spec as spec +import eth2spec.phase0.spec as spec -from pyspec.utils.minimal_ssz import signed_root -from pyspec.phase0.spec import ( +from eth2spec.utils.minimal_ssz import signed_root +from eth2spec.phase0.spec import ( # constants EMPTY_SIGNATURE, ZERO_HASH, @@ -28,10 +28,10 @@ verify_merkle_branch, hash, ) -from pyspec.phase0.state_transition import ( +from eth2spec.phase0.state_transition import ( state_transition, ) -from pyspec.utils.merkle_minimal import ( +from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 8ec57d56d6..fa7d1fb685 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -13,8 +13,8 @@ def build_spec(sourcefile, outfile): NewType, Tuple, ) -from pyspec.utils.minimal_ssz import * -from pyspec.utils.bls_stub import * +from eth2spec.utils.minimal_ssz import * +from eth2spec.utils.bls_stub import * """) diff --git a/test_generators/README.md b/test_generators/README.md index 51cca65612..bacb7229a7 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -113,7 +113,7 @@ if __name__ == "__main__": And to use the pyspec: ``` -from pyspec.phase0 import spec +from eth2spec.phase0 import spec ``` Recommendations: diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index 11ffa835a7..08042e7466 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -13,13 +13,13 @@ All the dynamic parts of the spec can be build at once with `make pyspec`. Alternatively, you can build a sub-set of the pyspec: `make phase0`. -Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/pyspec/phase0/spec.py` +Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py` ## Contributing Contributions are welcome, but consider implementing your idea as part of the spec itself first. The pyspec is not a replacement. -If you see opportunity to include any of the `pyspec/utils/` code in the spec, +If you see opportunity to include any of the `pyspec/eth2spec/utils/` code in the spec, please submit an issue or PR. ## License diff --git a/test_libs/pyspec/pyspec/__init__.py b/test_libs/pyspec/eth2spec/__init__.py similarity index 100% rename from test_libs/pyspec/pyspec/__init__.py rename to test_libs/pyspec/eth2spec/__init__.py diff --git a/test_libs/pyspec/pyspec/debug/__init__.py b/test_libs/pyspec/eth2spec/debug/__init__.py similarity index 100% rename from test_libs/pyspec/pyspec/debug/__init__.py rename to test_libs/pyspec/eth2spec/debug/__init__.py diff --git a/test_libs/pyspec/pyspec/debug/jsonize.py b/test_libs/pyspec/eth2spec/debug/jsonize.py similarity index 97% rename from test_libs/pyspec/pyspec/debug/jsonize.py rename to test_libs/pyspec/eth2spec/debug/jsonize.py index a77684543a..3ea6fe3f57 100644 --- a/test_libs/pyspec/pyspec/debug/jsonize.py +++ b/test_libs/pyspec/eth2spec/debug/jsonize.py @@ -1,4 +1,4 @@ -from pyspec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.minimal_ssz import hash_tree_root def jsonize(value, typ, include_hash_tree_roots=False): diff --git a/test_libs/pyspec/pyspec/phase0/__init__.py b/test_libs/pyspec/eth2spec/phase0/__init__.py similarity index 100% rename from test_libs/pyspec/pyspec/phase0/__init__.py rename to test_libs/pyspec/eth2spec/phase0/__init__.py diff --git a/test_libs/pyspec/pyspec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py similarity index 100% rename from test_libs/pyspec/pyspec/phase0/state_transition.py rename to test_libs/pyspec/eth2spec/phase0/state_transition.py diff --git a/test_libs/pyspec/pyspec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py similarity index 100% rename from test_libs/pyspec/pyspec/utils/__init__.py rename to test_libs/pyspec/eth2spec/utils/__init__.py diff --git a/test_libs/pyspec/pyspec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py similarity index 100% rename from test_libs/pyspec/pyspec/utils/bls_stub.py rename to test_libs/pyspec/eth2spec/utils/bls_stub.py diff --git a/test_libs/pyspec/pyspec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py similarity index 100% rename from test_libs/pyspec/pyspec/utils/hash_function.py rename to test_libs/pyspec/eth2spec/utils/hash_function.py diff --git a/test_libs/pyspec/pyspec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py similarity index 100% rename from test_libs/pyspec/pyspec/utils/merkle_minimal.py rename to test_libs/pyspec/eth2spec/utils/merkle_minimal.py diff --git a/test_libs/pyspec/pyspec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py similarity index 100% rename from test_libs/pyspec/pyspec/utils/minimal_ssz.py rename to test_libs/pyspec/eth2spec/utils/minimal_ssz.py From 9f329956930b66bb4a92390713c4c8136ab89fe4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 7 Apr 2019 16:32:48 +1000 Subject: [PATCH 20/61] enable generator to locate configurations --- test_libs/gen_helpers/gen_base/gen_runner.py | 10 +++++++++- test_libs/gen_helpers/gen_base/gen_typing.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test_libs/gen_helpers/gen_base/gen_runner.py b/test_libs/gen_helpers/gen_base/gen_runner.py index ad729449ac..4c3f69bb12 100644 --- a/test_libs/gen_helpers/gen_base/gen_runner.py +++ b/test_libs/gen_helpers/gen_base/gen_runner.py @@ -55,6 +55,13 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): default=False, help="if set overwrite test files if they exist", ) + parser.add_argument( + "-c", + "--configs-path", + dest="configs_path", + default=True, + help="specify the path of the configs directory (containing constants_presets and fork_timelines)", + ) args = parser.parse_args() output_dir = args.output_dir @@ -66,8 +73,9 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): yaml = YAML(pure=True) print(f"Generating tests for {generator_name}, creating {len(suite_creators)} test suite files...") + print(f"Reading config presets and fork timelines from {args.configs_path}") for suite_creator in suite_creators: - suite = suite_creator() + suite = suite_creator(args.configs_path) filename = make_filename_for_test(suite) path = output_dir / filename diff --git a/test_libs/gen_helpers/gen_base/gen_typing.py b/test_libs/gen_helpers/gen_base/gen_typing.py index 1384c870ff..d6bd679af3 100644 --- a/test_libs/gen_helpers/gen_base/gen_typing.py +++ b/test_libs/gen_helpers/gen_base/gen_typing.py @@ -2,4 +2,5 @@ TestCase = Dict[str, Any] TestSuite = Dict[str, Any] -TestSuiteCreator = Callable[[], TestSuite] +# Args: +TestSuiteCreator = Callable[[str], TestSuite] From 9eb640dd3b5f95f8a2e41ef49462cbb8a0bd58ab Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 7 Apr 2019 17:02:20 +1000 Subject: [PATCH 21/61] intro configuration support pkg --- scripts/phase0/build_spec.py | 21 ++++++++++++++----- scripts/phase0/function_puller.py | 3 ++- test_libs/config_helpers/README.md | 19 +++++++++++++++++ .../config_helpers/preset_loader/__init__.py | 0 .../config_helpers/preset_loader/loader.py | 18 ++++++++++++++++ test_libs/config_helpers/requirements.txt | 1 + test_libs/config_helpers/setup.py | 9 ++++++++ test_libs/gen_helpers/setup.py | 1 - 8 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 test_libs/config_helpers/README.md create mode 100644 test_libs/config_helpers/preset_loader/__init__.py create mode 100644 test_libs/config_helpers/preset_loader/loader.py create mode 100644 test_libs/config_helpers/requirements.txt create mode 100644 test_libs/config_helpers/setup.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index fa7d1fb685..98b1cad786 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -2,13 +2,14 @@ import function_puller -def build_spec(sourcefile, outfile): +def build_phase0_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" from typing import ( Any, Callable, + Dict, List, NewType, Tuple, @@ -41,7 +42,7 @@ def slot_to_epoch(x): return x // SLOTS_PER_EPOCH Store = None """) - code_lines += function_puller.get_lines(sourcefile) + code_lines += function_puller.get_spec(sourcefile) code_lines.append(""" # Monkey patch validator get committee code @@ -78,7 +79,16 @@ def hash(x): ret = _hash(x) hash_cache[x] = ret return ret - """) + +# Access to overwrite spec constants based on configuration +def apply_constants_preset(preset: Dict[str, Any]): + global_vars = globals() + for k, v in preset: + global_vars[k] = v + + # Deal with derived constants + GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) +""") with open(outfile, 'w') as out: out.write("\n".join(code_lines)) @@ -86,5 +96,6 @@ def hash(x): if __name__ == '__main__': if len(sys.argv) < 3: - print("Error: spec source and outfile must defined") - build_spec(sys.argv[1], sys.argv[2]) + print("Usage: ") + build_phase0_spec(sys.argv[1], sys.argv[2]) + diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 2cd0139c56..d0f3f66f10 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -1,7 +1,8 @@ import sys +from typing import List -def get_lines(file_name): +def get_spec(file_name) -> List[str]: code_lines = [] pulling_from = None current_name = None diff --git a/test_libs/config_helpers/README.md b/test_libs/config_helpers/README.md new file mode 100644 index 0000000000..1844820825 --- /dev/null +++ b/test_libs/config_helpers/README.md @@ -0,0 +1,19 @@ +# ETH 2.0 config helpers + +`preset_loader`: A util to load constants-presets with. +See [Constants-presets documentation](../../configs/constants_presets/README.md). + +Usage: + +```python +configs_path = 'configs/' + +... + +import preset_loader +from eth2spec.phase0 import spec +my_presets = preset_loader.load_presets(configs_path, 'main_net') +spec.apply_constants_preset(my_presets) +``` + +WARNING: this overwrites globals, make sure to prevent accidental collisions with other usage of the same imported specs package. diff --git a/test_libs/config_helpers/preset_loader/__init__.py b/test_libs/config_helpers/preset_loader/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/config_helpers/preset_loader/loader.py b/test_libs/config_helpers/preset_loader/loader.py new file mode 100644 index 0000000000..043c588050 --- /dev/null +++ b/test_libs/config_helpers/preset_loader/loader.py @@ -0,0 +1,18 @@ +from typing import Dict, Any + +from ruamel.yaml import ( + YAML, +) +from pathlib import Path +from os.path import join + + +def load_presets(configs_dir, presets_name) -> Dict[str, Any]: + """ + Loads the given preset + :param presets_name: The name of the generator. (lowercase snake_case) + :return: Dictionary, mapping of constant-name -> constant-value + """ + path = Path(join(configs_dir, 'constant_presets', presets_name+'.yaml')) + yaml = YAML(typ='safe') + return yaml.load(path) diff --git a/test_libs/config_helpers/requirements.txt b/test_libs/config_helpers/requirements.txt new file mode 100644 index 0000000000..e441a474b8 --- /dev/null +++ b/test_libs/config_helpers/requirements.txt @@ -0,0 +1 @@ +ruamel.yaml==0.15.87 diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py new file mode 100644 index 0000000000..90ad94ee44 --- /dev/null +++ b/test_libs/config_helpers/setup.py @@ -0,0 +1,9 @@ +from distutils.core import setup + +setup( + name='config_helpers', + packages=['preset_loader'], + install_requires=[ + "ruamel.yaml==0.15.87" + ] +) diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 88b971bf33..5de27a6dbe 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -2,7 +2,6 @@ setup( name='gen_helpers', - version='1.0', packages=['gen_base'], install_requires=[ "ruamel.yaml==0.15.87", From c350aaecf7ca1214ce196bcbd079751e389abd8c Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 7 Apr 2019 17:26:24 +1000 Subject: [PATCH 22/61] small format update, support new testing format in generator base pkg --- configs/fork_timelines/testing.yaml | 6 ++ specs/test_formats/README.md | 18 +++--- test_generators/README.md | 64 +++++++++++++-------- test_libs/gen_helpers/gen_base/gen_suite.py | 19 +++--- 4 files changed, 66 insertions(+), 41 deletions(-) create mode 100644 configs/fork_timelines/testing.yaml diff --git a/configs/fork_timelines/testing.yaml b/configs/fork_timelines/testing.yaml new file mode 100644 index 0000000000..957a53b8ca --- /dev/null +++ b/configs/fork_timelines/testing.yaml @@ -0,0 +1,6 @@ +# Testing fork timeline + +# Equal to GENESIS_EPOCH +phase0: 536870912 + +# No other forks considered in testing yet (to be implemented) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 371c489b65..f5d78193d4 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -89,14 +89,14 @@ The aim is to provide clients with a well-defined scope of work to run a particu ## Test Suite ``` -title: -- Display name for the test suite -summary: -- Summarizes the test suite -forks_timeline: -- Used to determine the forking timeline -forks: -- Runner decides what to do: run for each fork, or run for all at once, each fork transition, etc. - - ... -config: -- Used to determine which set of constants to run (possibly compile time) with -runner: *MUST be consistent with folder structure* -handler: *MUST be consistent with folder structure* +title: -- Display name for the test suite +summary: -- Summarizes the test suite +forks_timeline: -- Used to determine the forking timeline +forks: -- Runner decides what to do: run for each fork, or run for all at once, each fork transition, etc. + - ... +config: -- Used to determine which set of constants to run (possibly compile time) with +runner: *MUST be consistent with folder structure* +handler: *MUST be consistent with folder structure* test_cases: ... @@ -163,7 +163,7 @@ To prevent parsing of hundreds of different YAML files to test a specific test t ``` . <--- root of eth2.0 tests repository ├── bls <--- collection of handler for a specific test-runner, example runner: "bls" -│   ├── signing <--- collection of test suites for a specific handler, example handler: "signing". If no handler, use a dummy folder "main" +│   ├── signing <--- collection of test suites for a specific handler, example handler: "signing". If no multiple handlers, use a dummy folder (e.g. "main"), and specify that in the yaml. │   │   ├── sign_msg.yml <--- an entry list of test suites │   │   ... <--- more suite files (optional) │   ... <--- more handlers diff --git a/test_generators/README.md b/test_generators/README.md index bacb7229a7..c7c77c961e 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -59,11 +59,12 @@ Create a `requirements.txt` in the root of your generator directory: ``` eth-utils==1.4.1 ../../test_libs/gen_helpers -``` -And optionally, to include pyspec, add: -``` +../../test_libs/config_helpers ../../test_libs/pyspec ``` +The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself, to prevent code duplication and outdated tests. +Applying configurations to the spec is easy, and enables you to create test suites with different contexts. + Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement. Install all the necessary requirements (re-run when you add more): @@ -82,45 +83,60 @@ from eth_utils import ( to_dict, to_tuple ) +from preset_loader import loader +from eth2spec.phase0 import spec @to_dict -def bar_test_case(v: int): - yield "bar_v", v - yield "bar_v_plus_1", v + 1 - yield "bar_list", list(range(v)) +def example_test_case(v: int): + yield "spec_SHARD_COUNT", spec.SHARD_COUNT + yield "example", v @to_tuple -def generate_bar_test_cases(): +def generate_example_test_cases(): for i in range(10): - yield bar_test_case(i) + yield example_test_case(i) + +def example_minimal_suite(configs_path: str) -> gen_typing.TestSuite: + presets = loader.load_presets(configs_path, 'minimal') + spec.apply_constants_preset(presets) -def bar_test_suite() -> gen_typing.TestSuite: return gen_suite.render_suite( - title="bar_minimal", + title="example_minimal", summary="Minimal example suite, testing bar.", - fork="v0.5.1", + forks_timeline="testing", + forks=["phase0"], config="minimal", - test_cases=generate_bar_test_cases()) + handler="main", + test_cases=generate_example_test_cases()) -if __name__ == "__main__": - gen_runner.run_generator("foo", [bar_test_suite]) +def example_mainnet_suite(configs_path: str) -> gen_typing.TestSuite: + presets = loader.load_presets(configs_path, 'mainnet') + spec.apply_constants_preset(presets) -``` + return gen_suite.render_suite( + title="example_main_net", + summary="Main net based example suite.", + forks_timeline= "mainnet", + forks=["phase0"], + config="testing", + handler="main", + test_cases=generate_example_test_cases()) -And to use the pyspec: -``` -from eth2spec.phase0 import spec +if __name__ == "__main__": + gen_runner.run_generator("example", [example_minimal_suite, example_mainnet_suite]) ``` Recommendations: -- you can have more than just 1 generator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` +- you can have more than just 1 suite creator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` - you can concatenate lists of test cases, if you don't want to split it up in suites. -- you can split your suite generators into different python files/packages, good for code organization. -- use config "minimal" for performance. But also implement a suite with the default config where necessary +- you can split your suite creators into different python files/packages, good for code organization. +- use config "minimal" for performance. But also implement a suite with the default config where necessary. +- you may be able to write your test suite creator in a way where it does not make assumptions on constants. + If so, you can generate test suites with different configurations for the same scenario (see example). - the test-generator accepts `--output` and `--force` (overwrite output) ## How to add a new test generator @@ -133,13 +149,13 @@ In order to add a new test generator that builds `New Tests`: with any dependencies it may need. Leave it empty if your generator has none. 3. Your generator is assumed to have a `main.py` file in its root. By adding the base generator to your requirements, you can make a generator really easily. See docs below. -4. Your generator is called with `-o some/file/path/for_testing/can/be_anything`. +4. Your generator is called with `-o some/file/path/for_testing/can/be_anything -c some/other/path/to_configs/`. The base generator helps you handle this; you only have to define suite headers, and a list of tests for each suite you generate. 5. Finally, add any linting or testing commands to the [circleci config file](https://github.com/ethereum/eth2.0-test-generators/blob/master/.circleci/config.yml) if desired to increase code quality. - + Note: you do not have to change the makefile. However, if necessary (e.g. not using python, or mixing in other languages), submit an issue, and it can be a special case. Do note that generators should be easy to maintain, lean, and based on the spec. diff --git a/test_libs/gen_helpers/gen_base/gen_suite.py b/test_libs/gen_helpers/gen_base/gen_suite.py index fdfac8292c..3459d9ae35 100644 --- a/test_libs/gen_helpers/gen_base/gen_suite.py +++ b/test_libs/gen_helpers/gen_base/gen_suite.py @@ -1,17 +1,20 @@ from typing import Iterable -from eth_utils import ( - to_dict, -) - +from eth_utils import to_dict from gen_base.gen_typing import TestCase @to_dict -def render_suite(*, title: str, summary: str, fork: str, config: str, test_cases: Iterable[TestCase]): +def render_suite(*, + title: str, summary: str, + forks_timeline: str, forks: Iterable[str], + config: str, + handler: str, + test_cases: Iterable[TestCase]): yield "title", title - if summary is not None: - yield "summary", summary - yield "fork", fork + yield "summary", summary + yield "forks_timeline", forks_timeline, + yield "forks", forks yield "config", config + yield "handler", handler yield "test_cases", test_cases From 8fdae5bced7fb05fe6a5af6892862ef701e9719a Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 7 Apr 2019 17:50:39 +1000 Subject: [PATCH 23/61] support test format file structure in base generator --- test_libs/gen_helpers/gen_base/gen_runner.py | 50 +++++++++++++++----- test_libs/gen_helpers/gen_base/gen_typing.py | 6 ++- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/test_libs/gen_helpers/gen_base/gen_runner.py b/test_libs/gen_helpers/gen_base/gen_runner.py index 4c3f69bb12..8a179b0895 100644 --- a/test_libs/gen_helpers/gen_base/gen_runner.py +++ b/test_libs/gen_helpers/gen_base/gen_runner.py @@ -1,5 +1,5 @@ import argparse -import pathlib +from pathlib import Path import sys from typing import List @@ -10,14 +10,8 @@ from gen_base.gen_typing import TestSuiteCreator -def make_filename_for_test(test): - title = test["title"] - filename = title.lower().replace(" ", "_") + ".yaml" - return pathlib.Path(filename) - - def validate_output_dir(path_str): - path = pathlib.Path(path_str) + path = Path(path_str) if not path.exists(): raise argparse.ArgumentTypeError("Output directory must exist") @@ -28,6 +22,30 @@ def validate_output_dir(path_str): return path +def validate_configs_dir(path_str): + path = Path(path_str) + + if not path.exists(): + raise argparse.ArgumentTypeError("Configs directory must exist") + + if not path.is_dir(): + raise argparse.ArgumentTypeError("Config path must lead to a directory") + + if not Path(path, "constant_presets").exists(): + raise argparse.ArgumentTypeError("Constant Presets directory must exist") + + if not Path(path, "constant_presets").is_dir(): + raise argparse.ArgumentTypeError("Constant Presets path must lead to a directory") + + if not Path(path, "fork_timelines").exists(): + raise argparse.ArgumentTypeError("Fork Timelines directory must exist") + + if not Path(path, "fork_timelines").is_dir(): + raise argparse.ArgumentTypeError("Fork Timelines path must lead to a directory") + + return path + + def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): """ Implementation for a general test generator. @@ -59,7 +77,8 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): "-c", "--configs-path", dest="configs_path", - default=True, + required=True, + type=validate_configs_dir, help="specify the path of the configs directory (containing constants_presets and fork_timelines)", ) @@ -75,13 +94,18 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): print(f"Generating tests for {generator_name}, creating {len(suite_creators)} test suite files...") print(f"Reading config presets and fork timelines from {args.configs_path}") for suite_creator in suite_creators: - suite = suite_creator(args.configs_path) + (output_name, handler, suite) = suite_creator(args.configs_path) + + handler_output_dir = Path(output_dir) / Path(handler) + try: + handler_output_dir.mkdir() + except FileNotFoundError as e: + sys.exit(f'Error when creating handler dir {handler} for test "{suite["title"]}" ({e})') - filename = make_filename_for_test(suite) - path = output_dir / filename + out_path = handler_output_dir / Path(output_name + '.yaml') try: - with path.open(file_mode) as f: + with out_path.open(file_mode) as f: yaml.dump(suite, f) except IOError as e: sys.exit(f'Error when dumping test "{suite["title"]}" ({e})') diff --git a/test_libs/gen_helpers/gen_base/gen_typing.py b/test_libs/gen_helpers/gen_base/gen_typing.py index d6bd679af3..1cb3153154 100644 --- a/test_libs/gen_helpers/gen_base/gen_typing.py +++ b/test_libs/gen_helpers/gen_base/gen_typing.py @@ -1,6 +1,8 @@ -from typing import Callable, Dict, Any +from typing import Callable, Dict, Tuple, Any TestCase = Dict[str, Any] TestSuite = Dict[str, Any] +# Tuple: (output name, handler name, suite) -- output name excl. ".yaml" +TestSuiteOutput = Tuple[str, str, TestSuite] # Args: -TestSuiteCreator = Callable[[str], TestSuite] +TestSuiteCreator = Callable[[str], TestSuiteOutput] From fea40da6fe6dafabb5f38836ac7bd7c3d961abfb Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 7 Apr 2019 18:28:32 +1000 Subject: [PATCH 24/61] update generators readme example --- test_generators/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test_generators/README.md b/test_generators/README.md index c7c77c961e..bd509904c1 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -98,32 +98,32 @@ def generate_example_test_cases(): yield example_test_case(i) -def example_minimal_suite(configs_path: str) -> gen_typing.TestSuite: +def example_minimal_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'minimal') spec.apply_constants_preset(presets) - return gen_suite.render_suite( + return ("mini", "core", gen_suite.render_suite( title="example_minimal", summary="Minimal example suite, testing bar.", forks_timeline="testing", forks=["phase0"], config="minimal", handler="main", - test_cases=generate_example_test_cases()) + test_cases=generate_example_test_cases())) -def example_mainnet_suite(configs_path: str) -> gen_typing.TestSuite: +def example_mainnet_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'mainnet') spec.apply_constants_preset(presets) - return gen_suite.render_suite( + return ("full", "core", gen_suite.render_suite( title="example_main_net", summary="Main net based example suite.", forks_timeline= "mainnet", forks=["phase0"], config="testing", handler="main", - test_cases=generate_example_test_cases()) + test_cases=generate_example_test_cases())) if __name__ == "__main__": From db91c7fe9ed15b10de93f8ba2ea9157e0c37f508 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 7 Apr 2019 23:36:05 +1000 Subject: [PATCH 25/61] Work for new pyspec based test generators --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 6 +-- scripts/phase0/build_spec.py | 7 ++- scripts/phase0/function_puller.py | 26 +++++++--- .../config_helpers/preset_loader/loader.py | 11 +++- test_libs/gen_helpers/gen_base/gen_runner.py | 4 +- test_libs/pyspec/eth2spec/debug/decode.py | 28 ++++++++++ test_libs/pyspec/eth2spec/debug/encode.py | 26 ++++++++++ test_libs/pyspec/eth2spec/debug/jsonize.py | 52 ------------------- .../eth2spec/phase0/state_transition.py | 16 +++--- 10 files changed, 105 insertions(+), 73 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/debug/decode.py create mode 100644 test_libs/pyspec/eth2spec/debug/encode.py delete mode 100644 test_libs/pyspec/eth2spec/debug/jsonize.py diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 6eef9ad81d..27085d40ab 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -22,7 +22,7 @@ SHUFFLE_ROUND_COUNT: 90 # Deposit contract # --------------------------------------------------------------- # **TBD** -DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123567890123456789012357890 +DEPOSIT_CONTRACT_ADDRESS: 0x12345678901235678901234567890123567890 # 2**5 ` (= 32) DEPOSIT_CONTRACT_TREE_DEPTH: 32 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index e4c869dedc..f73531f380 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -22,7 +22,7 @@ SHUFFLE_ROUND_COUNT: 90 # Deposit contract # --------------------------------------------------------------- # **TBD** -DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123567890123456789012357890 +DEPOSIT_CONTRACT_ADDRESS: 0x12345678901235678901234567890123567890 # 2**5 ` (= 32) DEPOSIT_CONTRACT_TREE_DEPTH: 32 @@ -62,8 +62,8 @@ SLOTS_PER_EPOCH: 8 MIN_SEED_LOOKAHEAD: 1 # 2**2 ` (= 4) epochs 25.6 minutes ACTIVATION_EXIT_DELAY: 4 -# 2**4 ` (= 16) epochs ~1.7 hours -EPOCHS_PER_ETH1_VOTING_PERIOD: 16 +# [customized] higher frequency new deposits from eth1 for testing +EPOCHS_PER_ETH1_VOTING_PERIOD: 2 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 # 2**8 ` (= 256) epochs ~27 hours diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 98b1cad786..54adfdde72 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -83,11 +83,14 @@ def hash(x): # Access to overwrite spec constants based on configuration def apply_constants_preset(preset: Dict[str, Any]): global_vars = globals() - for k, v in preset: + for k, v in preset.items(): global_vars[k] = v # Deal with derived constants - GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT) + global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) + + # Initialize SSZ types again, to account for changed lengths + init_SSZ_types() """) with open(outfile, 'w') as out: diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index d0f3f66f10..fc7f9fb8c1 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -6,7 +6,8 @@ def get_spec(file_name) -> List[str]: code_lines = [] pulling_from = None current_name = None - processing_typedef = False + current_typedef = None + type_defs = [] for linenum, line in enumerate(open(sys.argv[1]).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -18,21 +19,26 @@ def get_spec(file_name) -> List[str]: if pulling_from is None: pulling_from = linenum else: - if processing_typedef: + if current_typedef is not None: assert code_lines[-1] == '}' code_lines[-1] = '})' + current_typedef[-1] = '})' + type_defs.append((current_name, current_typedef)) pulling_from = None - processing_typedef = False + current_typedef = None else: if pulling_from == linenum and line == '{': code_lines.append('%s = SSZType({' % current_name) - processing_typedef = True + current_typedef = ['global_vars["%s"] = SSZType({' % current_name] elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': - code_lines.append("") - code_lines.append("") + code_lines.append('') + code_lines.append('') code_lines.append(line) + # Remember type def lines + if current_typedef is not None: + current_typedef.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') if len(row) >= 2: @@ -48,4 +54,12 @@ def get_spec(file_name) -> List[str]: eligible = False if eligible: code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890'))) + # Build type-def re-initialization + code_lines.append('') + code_lines.append('def init_SSZ_types():') + code_lines.append(' global_vars = globals()') + for ssz_type_name, ssz_type in type_defs: + code_lines.append('') + for type_line in ssz_type: + code_lines.append(' ' + type_line) return code_lines diff --git a/test_libs/config_helpers/preset_loader/loader.py b/test_libs/config_helpers/preset_loader/loader.py index 043c588050..f37aca3936 100644 --- a/test_libs/config_helpers/preset_loader/loader.py +++ b/test_libs/config_helpers/preset_loader/loader.py @@ -14,5 +14,12 @@ def load_presets(configs_dir, presets_name) -> Dict[str, Any]: :return: Dictionary, mapping of constant-name -> constant-value """ path = Path(join(configs_dir, 'constant_presets', presets_name+'.yaml')) - yaml = YAML(typ='safe') - return yaml.load(path) + yaml = YAML(typ='base') + loaded = yaml.load(path) + out = dict() + for k, v in loaded.items(): + if v.startswith("0x"): + out[k] = bytes.fromhex(v[2:]) + else: + out[k] = int(v) + return out diff --git a/test_libs/gen_helpers/gen_base/gen_runner.py b/test_libs/gen_helpers/gen_base/gen_runner.py index 8a179b0895..e36d48b8b8 100644 --- a/test_libs/gen_helpers/gen_base/gen_runner.py +++ b/test_libs/gen_helpers/gen_base/gen_runner.py @@ -90,6 +90,7 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): file_mode = "w" yaml = YAML(pure=True) + yaml.default_flow_style = None print(f"Generating tests for {generator_name}, creating {len(suite_creators)} test suite files...") print(f"Reading config presets and fork timelines from {args.configs_path}") @@ -98,7 +99,8 @@ def run_generator(generator_name, suite_creators: List[TestSuiteCreator]): handler_output_dir = Path(output_dir) / Path(handler) try: - handler_output_dir.mkdir() + if not handler_output_dir.exists(): + handler_output_dir.mkdir() except FileNotFoundError as e: sys.exit(f'Error when creating handler dir {handler} for test "{suite["title"]}" ({e})') diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py new file mode 100644 index 0000000000..aeac3924d9 --- /dev/null +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -0,0 +1,28 @@ +from eth2spec.utils.minimal_ssz import hash_tree_root + + +def decode(json, typ): + if isinstance(typ, str) and typ[:4] == 'uint': + return json + elif typ == 'bool': + assert json in (True, False) + return json + elif isinstance(typ, list): + return [decode(element, typ[0]) for element in json] + elif isinstance(typ, str) and typ[:4] == 'byte': + return bytes.fromhex(json[2:]) + elif hasattr(typ, 'fields'): + temp = {} + for field, subtype in typ.fields.items(): + temp[field] = decode(json[field], subtype) + if field + "_hash_tree_root" in json: + assert(json[field + "_hash_tree_root"][2:] == + hash_tree_root(temp[field], subtype).hex()) + ret = typ(**temp) + if "hash_tree_root" in json: + assert(json["hash_tree_root"][2:] == + hash_tree_root(ret, typ).hex()) + return ret + else: + print(json, typ) + raise Exception("Type not recognized") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py new file mode 100644 index 0000000000..f50bc9d5e4 --- /dev/null +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -0,0 +1,26 @@ +from eth2spec.utils.minimal_ssz import hash_tree_root + + +def encode(value, typ, include_hash_tree_roots=False): + if isinstance(typ, str) and typ[:4] == 'uint': + return value + elif typ == 'bool': + assert value in (True, False) + return value + elif isinstance(typ, list): + return [encode(element, typ[0], include_hash_tree_roots) for element in value] + elif isinstance(typ, str) and typ[:4] == 'byte': + return '0x' + value.hex() + elif hasattr(typ, 'fields'): + ret = {} + for field, subtype in typ.fields.items(): + ret[field] = encode(getattr(value, field), subtype, include_hash_tree_roots) + if include_hash_tree_roots: + ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() + if include_hash_tree_roots: + ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() + return ret + else: + print(value, typ) + raise Exception("Type not recognized") + diff --git a/test_libs/pyspec/eth2spec/debug/jsonize.py b/test_libs/pyspec/eth2spec/debug/jsonize.py deleted file mode 100644 index 3ea6fe3f57..0000000000 --- a/test_libs/pyspec/eth2spec/debug/jsonize.py +++ /dev/null @@ -1,52 +0,0 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root - - -def jsonize(value, typ, include_hash_tree_roots=False): - if isinstance(typ, str) and typ[:4] == 'uint': - return value - elif typ == 'bool': - assert value in (True, False) - return value - elif isinstance(typ, list): - return [jsonize(element, typ[0], include_hash_tree_roots) for element in value] - elif isinstance(typ, str) and typ[:4] == 'byte': - return '0x' + value.hex() - elif hasattr(typ, 'fields'): - ret = {} - for field, subtype in typ.fields.items(): - ret[field] = jsonize(getattr(value, field), subtype, include_hash_tree_roots) - if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() - if include_hash_tree_roots: - ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() - return ret - else: - print(value, typ) - raise Exception("Type not recognized") - - -def dejsonize(json, typ): - if isinstance(typ, str) and typ[:4] == 'uint': - return json - elif typ == 'bool': - assert json in (True, False) - return json - elif isinstance(typ, list): - return [dejsonize(element, typ[0]) for element in json] - elif isinstance(typ, str) and typ[:4] == 'byte': - return bytes.fromhex(json[2:]) - elif hasattr(typ, 'fields'): - temp = {} - for field, subtype in typ.fields.items(): - temp[field] = dejsonize(json[field], subtype) - if field + "_hash_tree_root" in json: - assert(json[field + "_hash_tree_root"][2:] == - hash_tree_root(temp[field], subtype).hex()) - ret = typ(**temp) - if "hash_tree_root" in json: - assert(json["hash_tree_root"][2:] == - hash_tree_root(ret, typ).hex()) - return ret - else: - print(json, typ) - raise Exception("Type not recognized") diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py index 59ff07d370..d869fc408c 100644 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase0/state_transition.py @@ -99,13 +99,17 @@ def process_epoch_transition(state: BeaconState) -> None: spec.finish_epoch_update(state) -def state_transition(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: - while state.slot < block.slot: +def state_transition_to(state: BeaconState, up_to: int) -> BeaconState: + while state.slot < up_to: spec.cache_state(state) if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: process_epoch_transition(state) spec.advance_slot(state) - if block.slot == state.slot: - process_block(state, block, verify_state_root) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + state_transition_to(state, block.slot) + process_block(state, block, verify_state_root) + From e5fb9626e8a4e147d6ec9a00724e0b96877f5321 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 00:20:59 +1000 Subject: [PATCH 26/61] make ssz test gen conform to general test format --- test_generators/ssz/main.py | 116 ++++++------------ test_generators/ssz/renderers.py | 9 -- test_generators/ssz/requirements.txt | 4 +- ..._test_generators.py => uint_test_cases.py} | 34 ----- 4 files changed, 40 insertions(+), 123 deletions(-) rename test_generators/ssz/{uint_test_generators.py => uint_test_cases.py} (74%) diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py index d19ec12b48..f6454c3c09 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz/main.py @@ -1,84 +1,42 @@ -import argparse -import pathlib -import sys - -from ruamel.yaml import ( - YAML, -) - -from uint_test_generators import ( - generate_uint_bounds_test, - generate_uint_random_test, - generate_uint_wrong_length_test, +from uint_test_cases import ( + generate_random_uint_test_cases, + generate_uint_wrong_length_test_cases, + generate_uint_bounds_test_cases, + generate_uint_out_of_bounds_test_cases ) -test_generators = [ - generate_uint_random_test, - generate_uint_wrong_length_test, - generate_uint_bounds_test, -] - - -def make_filename_for_test(test): - title = test["title"] - filename = title.lower().replace(" ", "_") + ".yaml" - return pathlib.Path(filename) - - -def validate_output_dir(path_str): - path = pathlib.Path(path_str) - - if not path.exists(): - raise argparse.ArgumentTypeError("Output directory must exist") - - if not path.is_dir(): - raise argparse.ArgumentTypeError("Output path must lead to a directory") - - return path - - -parser = argparse.ArgumentParser( - prog="gen-ssz-tests", - description="Generate YAML test files for SSZ and tree hashing", -) -parser.add_argument( - "-o", - "--output-dir", - dest="output_dir", - required=True, - type=validate_output_dir, - help="directory into which the generated YAML files will be dumped" -) -parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - help="if set overwrite test files if they exist", -) +from gen_base import gen_runner, gen_suite, gen_typing + +def ssz_random_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("uint_random", "core", gen_suite.render_suite( + title="UInt Random", + summary="Random integers chosen uniformly over the allowed value range", + forks_timeline= "mainnet", + forks=["phase0"], + config="mainnet", + handler="core", + test_cases=generate_random_uint_test_cases())) + +def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("uint_wrong_length", "core", gen_suite.render_suite( + title="UInt Wrong Length", + summary="Serialized integers that are too short or too long", + forks_timeline= "mainnet", + forks=["phase0"], + config="mainnet", + handler="core", + test_cases=generate_uint_wrong_length_test_cases())) + +def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("uint_bounds", "core", gen_suite.render_suite( + title="UInt Bounds", + summary="Integers right at or beyond the bounds of the allowed value range", + forks_timeline= "mainnet", + forks=["phase0"], + config="mainnet", + handler="core", + test_cases=generate_uint_bounds_test_cases() + generate_uint_out_of_bounds_test_cases())) if __name__ == "__main__": - args = parser.parse_args() - output_dir = args.output_dir - if not args.force: - file_mode = "x" - else: - file_mode = "w" - - yaml = YAML(pure=True) - - print(f"generating {len(test_generators)} test files...") - for test_generator in test_generators: - test = test_generator() - - filename = make_filename_for_test(test) - path = output_dir / filename - - try: - with path.open(file_mode) as f: - yaml.dump(test, f) - except IOError as e: - sys.exit(f'Error when dumping test "{test["title"]}" ({e})') - - print("done.") + gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) \ No newline at end of file diff --git a/test_generators/ssz/renderers.py b/test_generators/ssz/renderers.py index e551ab14cf..ee8a928387 100644 --- a/test_generators/ssz/renderers.py +++ b/test_generators/ssz/renderers.py @@ -91,12 +91,3 @@ def render_test_case(*, sedes, valid, value=None, serial=None, description=None, if description is not None: yield description yield "tags", tags - - -@to_dict -def render_test(*, title, summary, fork, test_cases): - yield "title", title, - if summary is not None: - yield "summary", summary - yield "fork", fork - yield "test_cases", test_cases diff --git a/test_generators/ssz/requirements.txt b/test_generators/ssz/requirements.txt index 88193a01d6..94afc9d91b 100644 --- a/test_generators/ssz/requirements.txt +++ b/test_generators/ssz/requirements.txt @@ -1,2 +1,4 @@ -ruamel.yaml==0.15.87 +eth-utils==1.4.1 +../../test_libs/gen_helpers +../../test_libs/config_helpers ssz==0.1.0a2 diff --git a/test_generators/ssz/uint_test_generators.py b/test_generators/ssz/uint_test_cases.py similarity index 74% rename from test_generators/ssz/uint_test_generators.py rename to test_generators/ssz/uint_test_cases.py index c8c841fe72..d123564ca1 100644 --- a/test_generators/ssz/uint_test_generators.py +++ b/test_generators/ssz/uint_test_cases.py @@ -9,7 +9,6 @@ UInt, ) from renderers import ( - render_test, render_test_case, ) @@ -25,39 +24,6 @@ def get_random_bytes(length): return bytes(random.randint(0, 255) for _ in range(length)) -def generate_uint_bounds_test(): - test_cases = generate_uint_bounds_test_cases() + generate_uint_out_of_bounds_test_cases() - - return render_test( - title="UInt Bounds", - summary="Integers right at or beyond the bounds of the allowed value range", - fork="phase0-0.2.0", - test_cases=test_cases, - ) - - -def generate_uint_random_test(): - test_cases = generate_random_uint_test_cases() - - return render_test( - title="UInt Random", - summary="Random integers chosen uniformly over the allowed value range", - fork="phase0-0.2.0", - test_cases=test_cases, - ) - - -def generate_uint_wrong_length_test(): - test_cases = generate_uint_wrong_length_test_cases() - - return render_test( - title="UInt Wrong Length", - summary="Serialized integers that are too short or too long", - fork="phase0-0.2.0", - test_cases=test_cases, - ) - - @to_tuple def generate_random_uint_test_cases(): for bit_size in BIT_SIZES: From 41374957bb5e0211876f9603c89ec0497fcf2cb9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 01:52:51 +1000 Subject: [PATCH 27/61] update shuffling --- test_generators/shuffling/constants.py | 6 - test_generators/shuffling/core_helpers.py | 95 ------- test_generators/shuffling/main.py | 275 +++++++++------------ test_generators/shuffling/requirements.txt | 6 +- test_generators/shuffling/utils.py | 6 - test_generators/shuffling/yaml_objects.py | 25 -- 6 files changed, 124 insertions(+), 289 deletions(-) delete mode 100644 test_generators/shuffling/constants.py delete mode 100644 test_generators/shuffling/core_helpers.py delete mode 100644 test_generators/shuffling/utils.py delete mode 100644 test_generators/shuffling/yaml_objects.py diff --git a/test_generators/shuffling/constants.py b/test_generators/shuffling/constants.py deleted file mode 100644 index 92862f898d..0000000000 --- a/test_generators/shuffling/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -SLOTS_PER_EPOCH = 2**6 # 64 slots, 6.4 minutes -FAR_FUTURE_EPOCH = 2**64 - 1 # uint64 max -SHARD_COUNT = 2**10 # 1024 -TARGET_COMMITTEE_SIZE = 2**7 # 128 validators -ACTIVATION_EXIT_DELAY = 2**2 # 4 epochs -SHUFFLE_ROUND_COUNT = 90 diff --git a/test_generators/shuffling/core_helpers.py b/test_generators/shuffling/core_helpers.py deleted file mode 100644 index c424b771e5..0000000000 --- a/test_generators/shuffling/core_helpers.py +++ /dev/null @@ -1,95 +0,0 @@ -from typing import Any, List, NewType - -from constants import SLOTS_PER_EPOCH, SHARD_COUNT, TARGET_COMMITTEE_SIZE, SHUFFLE_ROUND_COUNT -from utils import hash -from yaml_objects import Validator - -Epoch = NewType("Epoch", int) -ValidatorIndex = NewType("ValidatorIndex", int) -Bytes32 = NewType("Bytes32", bytes) - - -def int_to_bytes1(x): - return x.to_bytes(1, 'little') - - -def int_to_bytes4(x): - return x.to_bytes(4, 'little') - - -def bytes_to_int(data: bytes) -> int: - return int.from_bytes(data, 'little') - - -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is active. - """ - return validator.activation_epoch <= epoch < validator.exit_epoch - - -def get_active_validator_indices(validators: List[Validator], epoch: Epoch) -> List[ValidatorIndex]: - """ - Get indices of active validators from ``validators``. - """ - return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)] - - -def split(values: List[Any], split_count: int) -> List[List[Any]]: - """ - Splits ``values`` into ``split_count`` pieces. - """ - list_length = len(values) - return [ - values[(list_length * i // split_count): (list_length * (i + 1) // split_count)] - for i in range(split_count) - ] - - -def get_epoch_committee_count(active_validator_count: int) -> int: - """ - Return the number of committees in one epoch. - """ - return max( - 1, - min( - SHARD_COUNT // SLOTS_PER_EPOCH, - active_validator_count // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - ) - ) * SLOTS_PER_EPOCH - - -def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int: - """ - Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. - - Utilizes 'swap or not' shuffling found in - https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf - See the 'generalized domain' algorithm on page 3. - """ - for round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size - flip = (pivot - index) % list_size - position = max(index, flip) - source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) - byte = source[(position % 256) // 8] - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return index - - -def get_shuffling(seed: Bytes32, - validators: List[Validator], - epoch: Epoch) -> List[List[ValidatorIndex]]: - """ - Shuffle active validators and split into crosslink committees. - Return a list of committees (each a list of validator indices). - """ - # Shuffle active validator indices - active_validator_indices = get_active_validator_indices(validators, epoch) - length = len(active_validator_indices) - shuffled_indices = [active_validator_indices[get_permuted_index(i, length, seed)] for i in range(length)] - - # Split the shuffled active validator indices - return split(shuffled_indices, get_epoch_committee_count(length)) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 03352944a4..e2edff7c7b 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -1,160 +1,127 @@ import random -import sys -import os -import yaml - -from constants import ACTIVATION_EXIT_DELAY, FAR_FUTURE_EPOCH -from core_helpers import get_shuffling -from yaml_objects import Validator - - -def noop(self, *args, **kw): - # Prevent !!str or !!binary tags - pass - - -yaml.emitter.Emitter.process_tag = noop - - -EPOCH = 1000 # The epoch, also a mean for the normal distribution - -# Standard deviation, around 8% validators will activate or exit within -# ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator -# shuffling -RAND_EPOCH_STD = 35 -MAX_EXIT_EPOCH = 5000 # Maximum exit_epoch for easier reading - - -def active_exited_validators_generator(): - """ - Random cases with variety of validator's activity status - """ - # Order not preserved - https://github.com/yaml/pyyaml/issues/110 - metadata = { - 'title': 'Shuffling Algorithm Tests 1', - 'summary': 'Test vectors for validator shuffling with different validator\'s activity status.' - ' Note: only relevant validator fields are defined.', - 'test_suite': 'shuffle', - 'fork': 'phase0-0.5.0', - } - - # Config - random.seed(int("0xEF00BEAC", 16)) - num_cases = 10 - - test_cases = [] - - for case in range(num_cases): - seedhash = bytes(random.randint(0, 255) for byte in range(32)) - idx_max = random.randint(128, 512) - - validators = [] - for idx in range(idx_max): - v = Validator(original_index=idx) - # 4/5 of all validators are active - if random.random() < 0.8: - # Choose a normally distributed epoch number - rand_epoch = round(random.gauss(EPOCH, RAND_EPOCH_STD)) - - # for 1/2 of *active* validators rand_epoch is the activation epoch +from eth2spec.phase0.spec import * +from eth_utils import ( + to_dict, to_tuple +) +from gen_base import gen_runner, gen_suite, gen_typing +from preset_loader import loader + + +@to_dict +def active_exited_validator_case(idx_max: int): + validators = [] + + # Standard deviation, around 8% validators will activate or exit within + # ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator + # shuffling + RAND_EPOCH_STD = 35 + + # TODO: fix epoch numbers + + slot = 1000 * SLOTS_PER_EPOCH + # The epoch, also a mean for the normal distribution + epoch = slot_to_epoch(slot) + MAX_EXIT_EPOCH = epoch + 5000 # Maximum exit_epoch for easier reading + + for idx in range(idx_max): + v = Validator( + pubkey=bytes(random.randint(0, 255) for _ in range(48)), + withdrawal_credentials=bytes(random.randint(0, 255) for _ in range(32)), + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + initiated_exit=False, + slashed=False, + high_balance=0 + ) + # 4/5 of all validators are active + if random.random() < 0.8: + # Choose a normally distributed epoch number + rand_epoch = round(random.gauss(epoch, RAND_EPOCH_STD)) + + # for 1/2 of *active* validators rand_epoch is the activation epoch + if random.random() < 0.5: + v.activation_epoch = rand_epoch + + # 1/4 of active validators will exit in forseeable future if random.random() < 0.5: - v.activation_epoch = rand_epoch - - # 1/4 of active validators will exit in forseeable future - if random.random() < 0.5: - v.exit_epoch = random.randint( - rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH) - # 1/4 of active validators in theory remain in the set indefinitely - else: - v.exit_epoch = FAR_FUTURE_EPOCH - # for the other active 1/2 rand_epoch is the exit epoch + v.exit_epoch = random.randint( + rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH) + # 1/4 of active validators in theory remain in the set indefinitely else: - v.activation_epoch = random.randint( - 0, rand_epoch - ACTIVATION_EXIT_DELAY) - v.exit_epoch = rand_epoch - - # The remaining 1/5 of all validators is not activated + v.exit_epoch = FAR_FUTURE_EPOCH + # for the other active 1/2 rand_epoch is the exit epoch else: - v.activation_epoch = FAR_FUTURE_EPOCH - v.exit_epoch = FAR_FUTURE_EPOCH - - validators.append(v) - - input_ = { - 'validators': validators, - 'epoch': EPOCH - } - output = get_shuffling( - seedhash, validators, input_['epoch']) - - test_cases.append({ - 'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output - }) - - return { - 'metadata': metadata, - 'filename': 'test_vector_shuffling.yml', - 'test_cases': test_cases - } - - -def validators_set_size_variety_generator(): - """ - Different validator set size cases, inspired by removed manual `permutated_index` tests - https://github.com/ethereum/eth2.0-test-generators/tree/bcd9ab2933d9f696901d1dfda0828061e9d3093f/permutated_index - """ - # Order not preserved - https://github.com/yaml/pyyaml/issues/110 - metadata = { - 'title': 'Shuffling Algorithm Tests 2', - 'summary': 'Test vectors for validator shuffling with different validator\'s set size.' - ' Note: only relevant validator fields are defined.', - 'test_suite': 'shuffle', - 'fork': 'tchaikovsky', - 'version': 1.0 - } - - # Config - random.seed(int("0xEF00BEAC", 16)) - - test_cases = [] - - seedhash = bytes(random.randint(0, 255) for byte in range(32)) - idx_max = 4096 - set_sizes = [1, 2, 3, 1024, idx_max] - - for size in set_sizes: - validators = [] - for idx in range(size): - v = Validator(original_index=idx) - v.activation_epoch = EPOCH + v.activation_epoch = random.randint( + 0, rand_epoch - ACTIVATION_EXIT_DELAY) + v.exit_epoch = rand_epoch + + # The remaining 1/5 of all validators is not activated + else: + v.activation_epoch = FAR_FUTURE_EPOCH v.exit_epoch = FAR_FUTURE_EPOCH - validators.append(v) - input_ = { - 'validators': validators, - 'epoch': EPOCH - } - output = get_shuffling( - seedhash, validators, input_['epoch']) - - test_cases.append({ - 'seed': '0x' + seedhash.hex(), 'input': input_, 'output': output - }) - - return { - 'metadata': metadata, - 'filename': 'shuffling_set_size.yml', - 'test_cases': test_cases - } - - -if __name__ == '__main__': - output_dir = sys.argv[2] - for generator in [active_exited_validators_generator, validators_set_size_variety_generator]: - result = generator() - filename = os.path.join(output_dir, result['filename']) - with open(filename, 'w') as outfile: - # Dump at top level - yaml.dump(result['metadata'], outfile, default_flow_style=False) - # default_flow_style will unravel "ValidatorRecord" and "committee" line, exploding file size - yaml.dump({'test_cases': result['test_cases']}, outfile) + + validators.append(v) + + query_slot = slot + random.randint(-1, 1) + state = get_genesis_beacon_state([], 0, None) + state.validator_registry = validators + state.latest_randao_mixes = [b'\xde\xad\xbe\xef' * 8 for _ in range(LATEST_RANDAO_MIXES_LENGTH)] + state.slot = slot + state.latest_start_shard = random.randint(0, SHARD_COUNT - 1) + randao_mix = bytes(random.randint(0, 255) for _ in range(32)) + state.latest_randao_mixes[slot_to_epoch(query_slot) % LATEST_RANDAO_MIXES_LENGTH] = randao_mix + + committees = get_crosslink_committees_at_slot(state, query_slot) + yield 'validator_registry', [ + { + 'activation_epoch': v.activation_epoch, + 'exit_epoch': v.exit_epoch + } for v in state.validator_registry + ] + yield 'randao_mix', '0x'+randao_mix.hex() + yield 'state_slot', state.slot + yield 'query_slot', query_slot + yield 'latest_start_shard', state.latest_start_shard + yield 'crosslink_committees', committees + + +@to_tuple +def active_exited_validator_cases(): + for i in range(3): + yield active_exited_validator_case(random.randint(100, min(200, SHARD_COUNT * 2))) + + +def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, 'minimal') + apply_constants_preset(presets) + + return ("shuffling_minimal", "core", gen_suite.render_suite( + title="Shuffling Algorithm Tests with minimal config", + summary="Test vectors for validator shuffling with different validator registry activity status and set size." + " Note: only relevant fields are defined.", + forks_timeline="testing", + forks=["phase0"], + config="minimal", + handler="core", + test_cases=active_exited_validator_cases())) + + +def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, 'mainnet') + apply_constants_preset(presets) + + return ("shuffling_full", "core", gen_suite.render_suite( + title="Shuffling Algorithm Tests with mainnet config", + summary="Test vectors for validator shuffling with different validator registry activity status and set size." + " Note: only relevant fields are defined.", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="core", + test_cases=active_exited_validator_cases())) + + +if __name__ == "__main__": + gen_runner.run_generator("shuffling", [mini_shuffling_suite, full_shuffling_suite]) diff --git a/test_generators/shuffling/requirements.txt b/test_generators/shuffling/requirements.txt index dde2fb67da..8f9bede8f3 100644 --- a/test_generators/shuffling/requirements.txt +++ b/test_generators/shuffling/requirements.txt @@ -1,4 +1,4 @@ -eth-hash[pycryptodome]==0.2.0 -eth-typing==2.0.0 eth-utils==1.4.1 -PyYAML==4.2b1 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec \ No newline at end of file diff --git a/test_generators/shuffling/utils.py b/test_generators/shuffling/utils.py deleted file mode 100644 index bcd2c6a3c1..0000000000 --- a/test_generators/shuffling/utils.py +++ /dev/null @@ -1,6 +0,0 @@ -from eth_typing import Hash32 -from eth_utils import keccak - - -def hash(x: bytes) -> Hash32: - return keccak(x) diff --git a/test_generators/shuffling/yaml_objects.py b/test_generators/shuffling/yaml_objects.py deleted file mode 100644 index 18e45220e6..0000000000 --- a/test_generators/shuffling/yaml_objects.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Any - -import yaml - - -class Validator(yaml.YAMLObject): - """ - A validator stub containing only the fields relevant for get_shuffling() - """ - fields = { - 'activation_epoch': 'uint64', - 'exit_epoch': 'uint64', - # Extra index field to ease testing/debugging - 'original_index': 'uint64', - } - - def __init__(self, **kwargs): - for k in self.fields.keys(): - setattr(self, k, kwargs.get(k)) - - def __setattr__(self, name: str, value: Any) -> None: - super().__setattr__(name, value) - - def __getattribute__(self, name: str) -> Any: - return super().__getattribute__(name) From d7b7640221acb9981dba0818e945ded0ccdd77bf Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 18:33:46 +1000 Subject: [PATCH 28/61] overhaul shuffling tests, focus on swap-or-not shuffle --- configs/constant_presets/minimal.yaml | 2 +- test_generators/shuffling/README.md | 26 +++--- test_generators/shuffling/main.py | 109 ++++---------------------- 3 files changed, 31 insertions(+), 106 deletions(-) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index f73531f380..8997bc5ed2 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -16,7 +16,7 @@ MAX_ATTESTATION_PARTICIPANTS: 4096 # 2**2 ` (= 4) MAX_EXIT_DEQUEUES_PER_EPOCH: 4 # See issue 563 -SHUFFLE_ROUND_COUNT: 90 +SHUFFLE_ROUND_COUNT: 10 # Deposit contract diff --git a/test_generators/shuffling/README.md b/test_generators/shuffling/README.md index 047a1b8721..5cf9d50e06 100644 --- a/test_generators/shuffling/README.md +++ b/test_generators/shuffling/README.md @@ -1,16 +1,16 @@ -# Shuffling Test Generator +# Shuffling Tests -``` -2018 Status Research & Development GmbH -Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). +Tests for the swap-or-not shuffling in ETH 2.0. -This work uses public domain work under CC0 from the Ethereum Foundation -https://github.com/ethereum/eth2.0-specs -``` +For implementers, possible test runners implementing testing can include: +1) just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation) +2) test un-permute-index (the reverse lookup. Implemented by running the shuffling rounds in reverse: from `round_count-1` to `0`) +3) test the optimized complete shuffle, where all indices are shuffled at once, test output in one go. +4) test complete shuffle in reverse (reverse rounds, same as 2) - -This file implements a test vectors generator for the shuffling algorithm described in the Ethereum -[specs](https://github.com/ethereum/eth2.0-specs/blob/2983e68f0305551083fac7fcf9330c1fc9da3411/specs/core/0_beacon-chain.md#get_new_shuffling) - -Utilizes 'swap or not' shuffling found in [An Enciphering Scheme Based on a Card Shuffle](https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf). -See the `Generalized domain` algorithm on page 3. +Tips for initial shuffling write: +- run with `round_count = 1` first, do the same with pyspec. +- start with permute index +- optimized shuffling implementations: + - vitalik, Python: https://github.com/ethereum/eth2.0-specs/pull/576#issue-250741806 + - protolambda, Go: https://github.com/protolambda/eth2-shuffle diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index e2edff7c7b..e8b2054a2a 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -1,6 +1,4 @@ -import random - -from eth2spec.phase0.spec import * +from eth2spec.phase0 import spec from eth_utils import ( to_dict, to_tuple ) @@ -9,118 +7,45 @@ @to_dict -def active_exited_validator_case(idx_max: int): - validators = [] - - # Standard deviation, around 8% validators will activate or exit within - # ENTRY_EXIT_DELAY inclusive from EPOCH thus creating an edge case for validator - # shuffling - RAND_EPOCH_STD = 35 - - # TODO: fix epoch numbers - - slot = 1000 * SLOTS_PER_EPOCH - # The epoch, also a mean for the normal distribution - epoch = slot_to_epoch(slot) - MAX_EXIT_EPOCH = epoch + 5000 # Maximum exit_epoch for easier reading - - for idx in range(idx_max): - v = Validator( - pubkey=bytes(random.randint(0, 255) for _ in range(48)), - withdrawal_credentials=bytes(random.randint(0, 255) for _ in range(32)), - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - initiated_exit=False, - slashed=False, - high_balance=0 - ) - # 4/5 of all validators are active - if random.random() < 0.8: - # Choose a normally distributed epoch number - rand_epoch = round(random.gauss(epoch, RAND_EPOCH_STD)) - - # for 1/2 of *active* validators rand_epoch is the activation epoch - if random.random() < 0.5: - v.activation_epoch = rand_epoch - - # 1/4 of active validators will exit in forseeable future - if random.random() < 0.5: - v.exit_epoch = random.randint( - rand_epoch + ACTIVATION_EXIT_DELAY + 1, MAX_EXIT_EPOCH) - # 1/4 of active validators in theory remain in the set indefinitely - else: - v.exit_epoch = FAR_FUTURE_EPOCH - # for the other active 1/2 rand_epoch is the exit epoch - else: - v.activation_epoch = random.randint( - 0, rand_epoch - ACTIVATION_EXIT_DELAY) - v.exit_epoch = rand_epoch - - # The remaining 1/5 of all validators is not activated - else: - v.activation_epoch = FAR_FUTURE_EPOCH - v.exit_epoch = FAR_FUTURE_EPOCH - - validators.append(v) - - query_slot = slot + random.randint(-1, 1) - state = get_genesis_beacon_state([], 0, None) - state.validator_registry = validators - state.latest_randao_mixes = [b'\xde\xad\xbe\xef' * 8 for _ in range(LATEST_RANDAO_MIXES_LENGTH)] - state.slot = slot - state.latest_start_shard = random.randint(0, SHARD_COUNT - 1) - randao_mix = bytes(random.randint(0, 255) for _ in range(32)) - state.latest_randao_mixes[slot_to_epoch(query_slot) % LATEST_RANDAO_MIXES_LENGTH] = randao_mix - - committees = get_crosslink_committees_at_slot(state, query_slot) - yield 'validator_registry', [ - { - 'activation_epoch': v.activation_epoch, - 'exit_epoch': v.exit_epoch - } for v in state.validator_registry - ] - yield 'randao_mix', '0x'+randao_mix.hex() - yield 'state_slot', state.slot - yield 'query_slot', query_slot - yield 'latest_start_shard', state.latest_start_shard - yield 'crosslink_committees', committees +def shuffling_case(seed: spec.Bytes32, count: int): + yield 'seed', '0x' + seed.hex() + yield 'count', count + yield 'shuffled', [spec.get_permuted_index(i, count, seed) for i in range(count)] @to_tuple -def active_exited_validator_cases(): - for i in range(3): - yield active_exited_validator_case(random.randint(100, min(200, SHARD_COUNT * 2))) +def shuffling_test_cases(): + for seed in [spec.hash(spec.int_to_bytes4(seed_init_value)) for seed_init_value in range(30)]: + for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000]: + yield shuffling_case(seed, count) def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'minimal') - apply_constants_preset(presets) + spec.apply_constants_preset(presets) return ("shuffling_minimal", "core", gen_suite.render_suite( - title="Shuffling Algorithm Tests with minimal config", - summary="Test vectors for validator shuffling with different validator registry activity status and set size." - " Note: only relevant fields are defined.", + title="Swap-or-Not Shuffling tests with minimal config", + summary="Swap or not shuffling, with minimally configured testing round-count", forks_timeline="testing", forks=["phase0"], config="minimal", handler="core", - test_cases=active_exited_validator_cases())) + test_cases=shuffling_test_cases())) def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'mainnet') - apply_constants_preset(presets) + spec.apply_constants_preset(presets) return ("shuffling_full", "core", gen_suite.render_suite( - title="Shuffling Algorithm Tests with mainnet config", - summary="Test vectors for validator shuffling with different validator registry activity status and set size." - " Note: only relevant fields are defined.", + title="Swap-or-Not Shuffling tests with mainnet config", + summary="Swap or not shuffling, with normal configured (secure) mainnet round-count", forks_timeline="mainnet", forks=["phase0"], config="mainnet", handler="core", - test_cases=active_exited_validator_cases())) + test_cases=shuffling_test_cases())) if __name__ == "__main__": From 9a0fd0afb927cba109bf8db4ec69df317b1dfc59 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 19:15:23 +1000 Subject: [PATCH 29/61] port BLS tests to new suite format --- test_generators/bls/main.py | 215 ++++++++++++++++----------- test_generators/bls/requirements.txt | 3 +- 2 files changed, 134 insertions(+), 84 deletions(-) diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 4c19c62491..0ffa0f66bb 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -2,16 +2,14 @@ BLS test vectors generator """ -# Standard library -import sys from typing import Tuple -# Third-party -import yaml -from py_ecc import bls +from eth_utils import ( + to_tuple, int_to_big_endian +) +from gen_base import gen_runner, gen_suite, gen_typing -# Ethereum -from eth_utils import int_to_big_endian, big_endian_to_int +from py_ecc import bls def int_to_hex(n: int) -> str: @@ -34,9 +32,9 @@ def hex_to_int(x: str) -> int: ] MESSAGES = [ - b'\x00' * 32, - b'\x56' * 32, - b'\xab' * 32, + bytes(b'\x00' * 32), + bytes(b'\x56' * 32), + bytes(b'\xab' * 32), ] PRIVKEYS = [ @@ -80,109 +78,160 @@ def hash_message_compressed(msg: bytes, domain: int) -> Tuple[str, str]: return [int_to_hex(z1), int_to_hex(z2)] -if __name__ == '__main__': - - # Order not preserved - https://github.com/yaml/pyyaml/issues/110 - metadata = { - 'title': 'BLS signature and aggregation tests', - 'summary': 'Test vectors for BLS signature', - 'test_suite': 'bls', - 'fork': 'phase0-0.5.0', - } - case01_message_hash_G2_uncompressed = [] +@to_tuple +def case01_message_hash_G2_uncompressed(): for msg in MESSAGES: for domain in DOMAINS: - case01_message_hash_G2_uncompressed.append({ - 'input': {'message': '0x' + msg.hex(), 'domain': int_to_hex(domain)}, + yield { + 'input': { + 'message': '0x' + msg.hex(), + 'domain': int_to_hex(domain) + }, 'output': hash_message(msg, domain) - }) + } - case02_message_hash_G2_compressed = [] +@to_tuple +def case02_message_hash_G2_compressed(): for msg in MESSAGES: for domain in DOMAINS: - case02_message_hash_G2_compressed.append({ - 'input': {'message': '0x' + msg.hex(), 'domain': int_to_hex(domain)}, + yield { + 'input': { + 'message': '0x' + msg.hex(), + 'domain': int_to_hex(domain) + }, 'output': hash_message_compressed(msg, domain) - }) + } - case03_private_to_public_key = [] - #  Used in later cases +@to_tuple +def case03_private_to_public_key(): pubkeys = [bls.privtopub(privkey) for privkey in PRIVKEYS] - #  Used in public key aggregation pubkeys_serial = ['0x' + pubkey.hex() for pubkey in pubkeys] - case03_private_to_public_key = [ - { + for privkey, pubkey_serial in zip(PRIVKEYS, pubkeys_serial): + yield { 'input': int_to_hex(privkey), 'output': pubkey_serial, } - for privkey, pubkey_serial in zip(PRIVKEYS, pubkeys_serial) - ] - case04_sign_messages = [] - sigs = [] # used in verify +@to_tuple +def case04_sign_messages(): for privkey in PRIVKEYS: for message in MESSAGES: for domain in DOMAINS: sig = bls.sign(message, privkey, domain) - case04_sign_messages.append({ + yield { 'input': { 'privkey': int_to_hex(privkey), 'message': '0x' + message.hex(), 'domain': int_to_hex(domain) }, 'output': '0x' + sig.hex() - }) - sigs.append(sig) + } + +# TODO: case05_verify_messages: Verify messages signed in case04 +# It takes too long, empty for now - # TODO: case05_verify_messages: Verify messages signed in case04 - # It takes too long, empty for now - case06_aggregate_sigs = [] +@to_tuple +def case06_aggregate_sigs(): for domain in DOMAINS: for message in MESSAGES: - sigs = [] - for privkey in PRIVKEYS: - sig = bls.sign(message, privkey, domain) - sigs.append(sig) - case06_aggregate_sigs.append({ + sigs = [bls.sign(message, privkey, domain) for privkey in PRIVKEYS] + yield { 'input': ['0x' + sig.hex() for sig in sigs], 'output': '0x' + bls.aggregate_signatures(sigs).hex(), - }) + } + +@to_tuple +def case07_aggregate_pubkeys(): + pubkeys = [bls.privtopub(privkey) for privkey in PRIVKEYS] + pubkeys_serial = ['0x' + pubkey.hex() for pubkey in pubkeys] + yield { + 'input': pubkeys_serial, + 'output': '0x' + bls.aggregate_pubkeys(pubkeys).hex(), + } - case07_aggregate_pubkeys = [ - { - 'input': pubkeys_serial, - 'output': '0x' + bls.aggregate_pubkeys(pubkeys).hex(), - } - ] - # TODO - # Aggregate verify - - # TODO - # Proof-of-possession - - with open(sys.argv[2] + "test_bls.yml", 'w') as outfile: - # Dump at top level - yaml.dump(metadata, outfile, default_flow_style=False) - # default_flow_style will unravel "ValidatorRecord" and "committee" line, - # exploding file size - yaml.dump( - {'case01_message_hash_G2_uncompressed': case01_message_hash_G2_uncompressed}, - outfile, - ) - yaml.dump( - {'case02_message_hash_G2_compressed': case02_message_hash_G2_compressed}, - outfile, - ) - yaml.dump( - {'case03_private_to_public_key': case03_private_to_public_key}, - outfile, - ) - yaml.dump({'case04_sign_messages': case04_sign_messages}, outfile) - - # Too time consuming to generate - # yaml.dump({'case05_verify_messages': case05_verify_messages}, outfile) - yaml.dump({'case06_aggregate_sigs': case06_aggregate_sigs}, outfile) - yaml.dump({'case07_aggregate_pubkeys': case07_aggregate_pubkeys}, outfile) +# TODO +# Aggregate verify + +# TODO +# Proof-of-possession + + +def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("g2_uncompressed", "msg_hash", gen_suite.render_suite( + title="BLS G2 Uncompressed msg hash", + summary="BLS G2 Uncompressed msg hash", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="msg_hash", + test_cases=case01_message_hash_G2_uncompressed())) + + +def bls_msg_hash_compressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("g2_compressed", "msg_hash", gen_suite.render_suite( + title="BLS G2 Compressed msg hash", + summary="BLS G2 Compressed msg hash", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="msg_hash", + test_cases=case02_message_hash_G2_compressed())) + + + +def bls_priv_to_pub_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("priv_to_pub", "priv_to_pub", gen_suite.render_suite( + title="BLS private key to pubkey", + summary="BLS Convert private key to public key", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="priv_to_pub", + test_cases=case03_private_to_public_key())) + + +def bls_sign_msg_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("sign_msg", "sign_msg", gen_suite.render_suite( + title="BLS sign msg", + summary="BLS Sign a message", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="sign_msg", + test_cases=case04_sign_messages())) + + +def bls_aggregate_sigs_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("aggregate_sigs", "aggregate_sigs", gen_suite.render_suite( + title="BLS aggregate sigs", + summary="BLS Aggregate signatures", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="aggregate_sigs", + test_cases=case06_aggregate_sigs())) + + +def bls_aggregate_pubkeys_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + return ("aggregate_pubkeys", "aggregate_pubkeys", gen_suite.render_suite( + title="BLS aggregate pubkeys", + summary="BLS Aggregate public keys", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="aggregate_pubkeys", + test_cases=case07_aggregate_pubkeys())) + + +if __name__ == "__main__": + gen_runner.run_generator("shuffling", [ + bls_msg_hash_compressed_suite, + bls_msg_hash_uncompressed_suite, + bls_priv_to_pub_suite, + bls_sign_msg_suite, + bls_aggregate_sigs_suite, + bls_aggregate_pubkeys_suite + ]) diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 3989a3a0fc..8a933d41ca 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,2 +1,3 @@ py-ecc==1.6.0 -PyYAML==4.2b1 +eth-utils==1.4.1 +../../test_libs/gen_helpers From a9054fb85d7a7798db4264471a39ba41db40ce79 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 19:17:53 +1000 Subject: [PATCH 30/61] fix BLS msg hash handler naming --- test_generators/bls/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 0ffa0f66bb..8b6949f9f6 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -160,24 +160,24 @@ def case07_aggregate_pubkeys(): def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("g2_uncompressed", "msg_hash", gen_suite.render_suite( + return ("g2_uncompressed", "msg_hash_compressed", gen_suite.render_suite( title="BLS G2 Uncompressed msg hash", summary="BLS G2 Uncompressed msg hash", forks_timeline="mainnet", forks=["phase0"], config="mainnet", - handler="msg_hash", + handler="msg_hash_compressed", test_cases=case01_message_hash_G2_uncompressed())) def bls_msg_hash_compressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("g2_compressed", "msg_hash", gen_suite.render_suite( + return ("g2_compressed", "msg_hash_uncompressed", gen_suite.render_suite( title="BLS G2 Compressed msg hash", summary="BLS G2 Compressed msg hash", forks_timeline="mainnet", forks=["phase0"], config="mainnet", - handler="msg_hash", + handler="msg_hash_uncompressed", test_cases=case02_message_hash_G2_compressed())) From 7e2b0a946fd2a6be08a8bb658cd2985f61007052 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 19:18:53 +1000 Subject: [PATCH 31/61] fix naming mistake --- test_generators/bls/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 8b6949f9f6..e5288c0454 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -160,24 +160,24 @@ def case07_aggregate_pubkeys(): def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("g2_uncompressed", "msg_hash_compressed", gen_suite.render_suite( + return ("g2_uncompressed", "msg_hash_uncompressed", gen_suite.render_suite( title="BLS G2 Uncompressed msg hash", summary="BLS G2 Uncompressed msg hash", forks_timeline="mainnet", forks=["phase0"], config="mainnet", - handler="msg_hash_compressed", + handler="msg_hash_uncompressed", test_cases=case01_message_hash_G2_uncompressed())) def bls_msg_hash_compressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("g2_compressed", "msg_hash_uncompressed", gen_suite.render_suite( + return ("g2_compressed", "msg_hash_compressed", gen_suite.render_suite( title="BLS G2 Compressed msg hash", summary="BLS G2 Compressed msg hash", forks_timeline="mainnet", forks=["phase0"], config="mainnet", - handler="msg_hash_uncompressed", + handler="msg_hash_compressed", test_cases=case02_message_hash_G2_compressed())) From f958adbff134c91d270f2a689c4981b41457d64c Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 11 Apr 2019 19:25:00 +1000 Subject: [PATCH 32/61] generator running fixes --- Makefile | 3 ++- test_generators/bls/main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bf23a14427..7ca2cc909b 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec PY_TEST_DIR = ./py_tests YAML_TEST_DIR = ./yaml_tests GENERATOR_DIR = ./test_generators +CONFIGS_DIR = ./configs # Collect a list of generator names GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/))) @@ -57,7 +58,7 @@ define build_yaml_tests # 3) Install all the necessary requirements # 4) Run the generator. The generator is assumed to have an "main.py" file. # 5) We output to the tests dir (generator program should accept a "-o " argument. - cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) + cd $(GENERATOR_DIR)$(1); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements.txt; python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR) $(info generator $(1) finished) endef diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index e5288c0454..1a03730685 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -227,7 +227,7 @@ def bls_aggregate_pubkeys_suite(configs_path: str) -> gen_typing.TestSuiteOutput if __name__ == "__main__": - gen_runner.run_generator("shuffling", [ + gen_runner.run_generator("bls", [ bls_msg_hash_compressed_suite, bls_msg_hash_uncompressed_suite, bls_priv_to_pub_suite, From 5824117cf9b3c606bd145f2fc98963f4fdcb3289 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 00:53:32 +1000 Subject: [PATCH 33/61] start to implement operations testing --- test_generators/operations/README.md | 13 ++ test_generators/operations/deposit.py | 173 ++++++++++++++++++++ test_generators/operations/genesis.py | 44 +++++ test_generators/operations/keys.py | 7 + test_generators/operations/main.py | 9 + test_generators/operations/requirements.txt | 5 + 6 files changed, 251 insertions(+) create mode 100644 test_generators/operations/README.md create mode 100644 test_generators/operations/deposit.py create mode 100644 test_generators/operations/genesis.py create mode 100644 test_generators/operations/keys.py create mode 100644 test_generators/operations/main.py create mode 100644 test_generators/operations/requirements.txt diff --git a/test_generators/operations/README.md b/test_generators/operations/README.md new file mode 100644 index 0000000000..9f1ecfddb8 --- /dev/null +++ b/test_generators/operations/README.md @@ -0,0 +1,13 @@ +# Operations + +Operations (or "transactions" in previous spec iterations), + are atomic changes to the state, introduced by embedding in blocks. + +This generators provides a series of test suites, divided into handler, for each operation type. +A operation test-runner can consume these operation test-suites, + and handle different kinds of operations by processing the cases using the specified test handler. + +Information on the format of the tests can be found in the [operations test formats documentation](../../specs/test_formats/operations/README.md). + + + diff --git a/test_generators/operations/deposit.py b/test_generators/operations/deposit.py new file mode 100644 index 0000000000..a72b1fbaa6 --- /dev/null +++ b/test_generators/operations/deposit.py @@ -0,0 +1,173 @@ +from eth2spec.phase0 import spec +from eth_utils import ( + to_dict, to_tuple +) +from gen_base import gen_suite, gen_typing +from preset_loader import loader +from eth2spec.debug.encode import encode +from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof + +from typing import List, Tuple + +import genesis +import keys +from py_ecc import bls + + +def build_deposit_data(state, + pubkey: spec.BLSPubkey, + withdrawal_cred: spec.Bytes32, + privkey: int, + amount: int): + deposit_data = spec.DepositData( + pubkey=pubkey, + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[1:], + amount=amount, + proof_of_possession=spec.EMPTY_SIGNATURE, + ) + deposit_data.proof_of_possession = bls.sign( + message_hash=signing_root(deposit_data), + privkey=privkey, + domain=spec.get_domain( + state.fork, + spec.get_current_epoch(state), + spec.DOMAIN_DEPOSIT, + ) + ) + return deposit_data + + +def build_deposit(state, + deposit_data_leaves: List[spec.Bytes32], + pubkey: spec.BLSPubkey, + withdrawal_cred: spec.Bytes32, + privkey: int, + amount: int) -> spec.Deposit: + + deposit_data = build_deposit_data(state, pubkey, withdrawal_cred, privkey, amount) + + item = spec.hash(deposit_data.serialize()) + index = len(deposit_data_leaves) + deposit_data_leaves.append(item) + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + proof = list(get_merkle_proof(tree, item_index=index)) + + deposit = spec.Deposit( + proof=list(proof), + index=index, + data=deposit_data, + ) + + return deposit + + +def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[spec.Deposit, spec.BeaconState, List[spec.Bytes32]]: + genesis_deposits = genesis.create_deposits( + keys.pubkeys[:initial_validator_count], + keys.withdrawal_creds[:initial_validator_count] + ) + state = genesis.create_genesis_state(genesis_deposits) + + deposit_data_leaves = [spec.hash(dep.data.serialize()) for dep in genesis_deposits] + + deposit = build_deposit( + state, + deposit_data_leaves, + keys.pubkeys[index], + keys.withdrawal_creds[index], + keys.privkeys[index], + spec.MAX_DEPOSIT_AMOUNT, + ) + + state.latest_eth1_data.deposit_root = get_merkle_root(tuple(deposit_data_leaves)) + state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + + return deposit, state, deposit_data_leaves + + +@to_dict +def valid_deposit(): + new_dep, state, leaves = build_deposit_for_index(10, 10) + state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) + state.latest_eth1_data.deposit_count = len(leaves) + yield 'case', 'valid deposit to add new validator' + yield 'pre', encode(state, spec.BeaconState) + yield 'deposit', encode(new_dep, spec.Deposit) + spec.process_deposit(state, new_dep) + yield 'post', encode(state, spec.BeaconState) + + +@to_dict +def valid_topup(): + new_dep, state, leaves = build_deposit_for_index(10, 3) + state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) + state.latest_eth1_data.deposit_count = len(leaves) + yield 'case', 'valid deposit to top-up existing validator' + yield 'pre', encode(state, spec.BeaconState) + yield 'deposit', encode(new_dep, spec.Deposit) + spec.process_deposit(state, new_dep) + yield 'post', encode(state, spec.BeaconState) + + +@to_dict +def invalid_deposit_index(): + new_dep, state, leaves = build_deposit_for_index(10, 10) + state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) + state.latest_eth1_data.deposit_count = len(leaves) + # Mess up deposit index, 1 too small + state.deposit_index = 9 + + yield 'case', 'invalid deposit index' + yield 'pre', encode(state, spec.BeaconState) + yield 'deposit', encode(new_dep, spec.Deposit) + yield 'post', None + +@to_dict +def invalid_deposit_proof(): + new_dep, state, leaves = build_deposit_for_index(10, 10) + state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) + state.latest_eth1_data.deposit_count = len(leaves) + # Make deposit proof invalid (at bottom of proof) + new_dep.proof[-1] = spec.ZERO_HASH + + yield 'case', 'invalid deposit proof' + yield 'pre', encode(state, spec.BeaconState) + yield 'deposit', encode(new_dep, spec.Deposit) + yield 'post', None + + +@to_tuple +def deposit_cases(): + yield valid_deposit() + yield valid_topup() + yield invalid_deposit_index() + yield invalid_deposit_proof() + + +def mini_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, 'minimal') + spec.apply_constants_preset(presets) + + return ("deposit_minimal", "deposits", gen_suite.render_suite( + title="deposit operation", + summary="Test suite for deposit type operation processing", + forks_timeline="testing", + forks=["phase0"], + config="minimal", + handler="core", + test_cases=deposit_cases())) + + +def full_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput: + presets = loader.load_presets(configs_path, 'mainnet') + spec.apply_constants_preset(presets) + + return ("deposit_full", "deposits", gen_suite.render_suite( + title="deposit operation", + summary="Test suite for deposit type operation processing", + forks_timeline="mainnet", + forks=["phase0"], + config="mainnet", + handler="core", + test_cases=deposit_cases())) diff --git a/test_generators/operations/genesis.py b/test_generators/operations/genesis.py new file mode 100644 index 0000000000..7e0146f67a --- /dev/null +++ b/test_generators/operations/genesis.py @@ -0,0 +1,44 @@ +from eth2spec.phase0 import spec +from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof +from typing import List + + +def create_genesis_state(deposits: List[spec.Deposit]) -> spec.BeaconState: + deposit_root = get_merkle_root((tuple([spec.hash(dep.data.serialize()) for dep in deposits]))) + + return spec.get_genesis_beacon_state( + deposits, + genesis_time=0, + genesis_eth1_data=spec.Eth1Data( + deposit_root=deposit_root, + deposit_count=len(deposits), + block_hash=spec.ZERO_HASH, + ), + ) + + +def create_deposits(pubkeys: List[spec.BLSPubkey], withdrawal_cred: List[spec.Bytes32]) -> List[spec.Deposit]: + + # Mock proof of possession + proof_of_possession = b'\x33' * 96 + + deposit_data = [ + spec.DepositData( + pubkey=pubkeys[i], + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + withdrawal_cred[i][1:], + amount=spec.MAX_DEPOSIT_AMOUNT, + proof_of_possession=proof_of_possession, + ) for i in range(len(pubkeys)) + ] + + # Fill tree with existing deposits + deposit_data_leaves = [spec.hash(data.serialize()) for data in deposit_data] + tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) + + return [ + spec.Deposit( + proof=list(get_merkle_proof(tree, item_index=i)), + index=i, + data=deposit_data[i] + ) for i in range(len(deposit_data)) + ] diff --git a/test_generators/operations/keys.py b/test_generators/operations/keys.py new file mode 100644 index 0000000000..db4f59e0e6 --- /dev/null +++ b/test_generators/operations/keys.py @@ -0,0 +1,7 @@ +from py_ecc import bls +from eth2spec.phase0.spec import hash + +privkeys = list(range(1, 101)) +pubkeys = [bls.privtopub(k) for k in privkeys] +# Insecure, but easier to follow +withdrawal_creds = [hash(bls.privtopub(k)) for k in privkeys] diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py new file mode 100644 index 0000000000..dd934c7580 --- /dev/null +++ b/test_generators/operations/main.py @@ -0,0 +1,9 @@ +from gen_base import gen_runner + +from deposit import mini_deposits_suite, full_deposits_suite + +if __name__ == "__main__": + gen_runner.run_generator("operations", [ + mini_deposits_suite, + full_deposits_suite + ]) diff --git a/test_generators/operations/requirements.txt b/test_generators/operations/requirements.txt new file mode 100644 index 0000000000..dfe8535365 --- /dev/null +++ b/test_generators/operations/requirements.txt @@ -0,0 +1,5 @@ +eth-utils==1.4.1 +../../test_libs/gen_helpers +../../test_libs/config_helpers +../../test_libs/pyspec +py_ecc \ No newline at end of file From 7ca20d71cac78c6a6084c9c8e184dde223dab5d7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 22:15:30 +1000 Subject: [PATCH 34/61] minor naming tweaks, document BLS and deposit test formats --- specs/test_formats/README.md | 8 +++++- specs/test_formats/bls/README.md | 15 +++++++++++ specs/test_formats/bls/aggregate_pubkeys.md | 21 +++++++++++++++ specs/test_formats/bls/aggregate_sigs.md | 21 +++++++++++++++ .../bls/msg_hash_g2_compressed.md | 23 ++++++++++++++++ .../bls/msg_hash_g2_uncompressed.md | 23 ++++++++++++++++ specs/test_formats/bls/priv_to_pub.md | 21 +++++++++++++++ specs/test_formats/bls/sign_msg.md | 24 +++++++++++++++++ specs/test_formats/operations/README.md | 4 +++ specs/test_formats/operations/deposits.md | 26 +++++++++++++++++++ specs/test_formats/shuffling/README.md | 0 specs/test_formats/ssz/README.md | 0 test_generators/bls/main.py | 4 +-- .../operations/{deposit.py => deposits.py} | 0 test_generators/operations/main.py | 2 +- 15 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 specs/test_formats/bls/README.md create mode 100644 specs/test_formats/bls/aggregate_pubkeys.md create mode 100644 specs/test_formats/bls/aggregate_sigs.md create mode 100644 specs/test_formats/bls/msg_hash_g2_compressed.md create mode 100644 specs/test_formats/bls/msg_hash_g2_uncompressed.md create mode 100644 specs/test_formats/bls/priv_to_pub.md create mode 100644 specs/test_formats/bls/sign_msg.md create mode 100644 specs/test_formats/operations/README.md create mode 100644 specs/test_formats/operations/deposits.md create mode 100644 specs/test_formats/shuffling/README.md create mode 100644 specs/test_formats/ssz/README.md rename test_generators/operations/{deposit.py => deposits.py} (100%) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index f5d78193d4..f64b93fa30 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -75,7 +75,13 @@ The second is still somewhat ambiguous: some tests may want cover multiple forks There is a common factor here however: the options are exclusive, and give a clear idea on what test suites need to be ran to cover testing for a specific fork. The way this list of forks is interpreted, is up to the test-runner: State-transition test suites may want to just declare forks that are being covered in the test suite, - whereas shuffling test suites may want to declare a list of forks to test the shuffling algorithm for individually. + whereas shuffling test suites may want to declare a list of forks to test the shuffling algorithm for individually. + +Test-formats specify the following `forks` interpretation rules: + +- `collective`: the test suite applies to all specified forks, and only needs to run once +- `individual`: the test suite should be ran against every fork +- more types may be specified with future test types. ### Test completeness diff --git a/specs/test_formats/bls/README.md b/specs/test_formats/bls/README.md new file mode 100644 index 0000000000..700686bdd7 --- /dev/null +++ b/specs/test_formats/bls/README.md @@ -0,0 +1,15 @@ +# BLS test suite + +A test suite for BLS. Primarily geared towards verifying the *integration* of any BLS library. +We do not recommend to roll your own crypto, or use an untested BLS library. + +The BLS test suite runner has the following handlers: + +- [`aggregate_pubkeys`](./aggregate_pubkeys.md) +- [`aggregate_sigs`](./aggregate_sigs.md) +- [`msg_hash_g2_compressed`](./msg_hash_g2_compressed.md) +- [`msg_hash_g2_uncompressed`](./msg_hash_g2_uncompressed.md) +- [`priv_to_pub`](./priv_to_pub.md) +- [`sign_msg`](./sign_msg.md) + +Note: signature-verification and aggregate-verify test cases are not yet supported. diff --git a/specs/test_formats/bls/aggregate_pubkeys.md b/specs/test_formats/bls/aggregate_pubkeys.md new file mode 100644 index 0000000000..9a6f1cc256 --- /dev/null +++ b/specs/test_formats/bls/aggregate_pubkeys.md @@ -0,0 +1,21 @@ +# Test format: BLS pubkey aggregation + +A BLS pubkey aggregation combines a series of pubkeys into a single pubkey. + +## Test case format + +```yaml +input: List[BLS Pubkey] -- list of input BLS pubkeys +output: BLS Pubkey -- expected output, single BLS pubkey +``` + +`BLS Pubkey` here is encoded as a string: hexadecimal encoding of 48 bytes (96 nibbles), prefixed with `0x`. + + +## Condition + +The `aggregate_pubkeys` handler should aggregate the keys in the `input`, and the result should match the expected `output`. + +## Forks + +Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/aggregate_sigs.md b/specs/test_formats/bls/aggregate_sigs.md new file mode 100644 index 0000000000..1588e26cba --- /dev/null +++ b/specs/test_formats/bls/aggregate_sigs.md @@ -0,0 +1,21 @@ +# Test format: BLS signature aggregation + +A BLS signature aggregation combines a series of signatures into a single signature. + +## Test case format + +```yaml +input: List[BLS Signature] -- list of input BLS signatures +output: BLS Signature -- expected output, single BLS signature +``` + +`BLS Signature` here is encoded as a string: hexadecimal encoding of 96 bytes (192 nibbles), prefixed with `0x`. + + +## Condition + +The `aggregate_sigs` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. + +## Forks + +Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/specs/test_formats/bls/msg_hash_g2_compressed.md new file mode 100644 index 0000000000..51c64e28b4 --- /dev/null +++ b/specs/test_formats/bls/msg_hash_g2_compressed.md @@ -0,0 +1,23 @@ +# Test format: BLS hash-compressed + +A BLS compressed-hash to G2. + +## Test case format + +```yaml +input: + message: bytes32, + domain: bytes -- any number +output: List[bytes48] -- length of two +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` + + +## Condition + +The `msg_hash_g2_compressed` handler should hash the `message`, with the given `domain`, to G2 with compression, and the result should match the expected `output`. + +## Forks + +Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/specs/test_formats/bls/msg_hash_g2_uncompressed.md new file mode 100644 index 0000000000..b7d2caa02b --- /dev/null +++ b/specs/test_formats/bls/msg_hash_g2_uncompressed.md @@ -0,0 +1,23 @@ +# Test format: BLS hash-uncompressed + +A BLS uncompressed-hash to G2. + +## Test case format + +```yaml +input: + message: bytes32, + domain: bytes -- any number +output: List[List[bytes48]] -- 3 lists, each a length of two +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` + + +## Condition + +The `msg_hash_g2_uncompressed` handler should hash the `message`, with the given `domain`, to G2, without compression, and the result should match the expected `output`. + +## Forks + +Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/priv_to_pub.md b/specs/test_formats/bls/priv_to_pub.md new file mode 100644 index 0000000000..9265b83ed3 --- /dev/null +++ b/specs/test_formats/bls/priv_to_pub.md @@ -0,0 +1,21 @@ +# Test format: BLS private key to pubkey + +A BLS private key to public key conversion. + +## Test case format + +```yaml +input: bytes32 -- the private key +output: bytes48 -- the public key +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` + + +## Condition + +The `priv_to_pub` handler should compute the public key for the given private key `input`, and the result should match the expected `output`. + +## Forks + +Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/sign_msg.md b/specs/test_formats/bls/sign_msg.md new file mode 100644 index 0000000000..3a6d63fa27 --- /dev/null +++ b/specs/test_formats/bls/sign_msg.md @@ -0,0 +1,24 @@ +# Test format: BLS sign message + +Message signing with BLS should produce a signature. + +## Test case format + +```yaml +input: + privkey: bytes32 -- the private key used for signing + message: bytes32 -- input message to sign (a hash) + domain: bytes -- BLS domain +output: bytes96 -- expected signature +``` + +All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x` + + +## Condition + +The `sign_msg` handler should sign the given `message`, with `domain`, using the given `privkey`, and the result should match the expected `output`. + +## Forks + +Forks-interpretation: `collective` diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md new file mode 100644 index 0000000000..747d8217c4 --- /dev/null +++ b/specs/test_formats/operations/README.md @@ -0,0 +1,4 @@ +# Operations test suite + +The operations test suite + diff --git a/specs/test_formats/operations/deposits.md b/specs/test_formats/operations/deposits.md new file mode 100644 index 0000000000..5aaed24f79 --- /dev/null +++ b/specs/test_formats/operations/deposits.md @@ -0,0 +1,26 @@ +# Test format: Deposit operations + +A deposit is a form of an operation (or "transaction"), modifying the state. + +## Test case format + +```yaml +case: string -- description of test case, purely for debugging purposes +pre: BeaconState -- state before applying the deposit +deposit: Deposit -- the deposit +post: BeaconState -- state after applying the deposit. No value if deposit processing is aborted. +``` + +## Condition + +A `deposits` handler of the `operations` should process these cases, + calling the implementation of the `process_deposit(state, deposit)` functionality described in the spec. +The resulting state should match the expected `post` state, or no change if the `post` state is left blank. + +## Forks + +Forks-interpretation: `collective` + +Pre and post state contain slot numbers, and are time sensitive. +Additional tests will be added for future forks to cover fork-specific behavior based on input data + (including suites with deposits on fork transition blocks, covering multiple forks) diff --git a/specs/test_formats/shuffling/README.md b/specs/test_formats/shuffling/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/specs/test_formats/ssz/README.md b/specs/test_formats/ssz/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 1a03730685..da6a79aaed 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -160,7 +160,7 @@ def case07_aggregate_pubkeys(): def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("g2_uncompressed", "msg_hash_uncompressed", gen_suite.render_suite( + return ("g2_uncompressed", "msg_hash_g2_uncompressed", gen_suite.render_suite( title="BLS G2 Uncompressed msg hash", summary="BLS G2 Uncompressed msg hash", forks_timeline="mainnet", @@ -171,7 +171,7 @@ def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOu def bls_msg_hash_compressed_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("g2_compressed", "msg_hash_compressed", gen_suite.render_suite( + return ("g2_compressed", "msg_hash_g2_compressed", gen_suite.render_suite( title="BLS G2 Compressed msg hash", summary="BLS G2 Compressed msg hash", forks_timeline="mainnet", diff --git a/test_generators/operations/deposit.py b/test_generators/operations/deposits.py similarity index 100% rename from test_generators/operations/deposit.py rename to test_generators/operations/deposits.py diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index dd934c7580..8b0a2a6d83 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -1,6 +1,6 @@ from gen_base import gen_runner -from deposit import mini_deposits_suite, full_deposits_suite +from deposits import mini_deposits_suite, full_deposits_suite if __name__ == "__main__": gen_runner.run_generator("operations", [ From fcb7e3437e9902e9443bd2f1ce113e2df8d3b864 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 22:19:10 +1000 Subject: [PATCH 35/61] test format doc tweaks --- specs/test_formats/bls/README.md | 4 ++-- specs/test_formats/operations/README.md | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/specs/test_formats/bls/README.md b/specs/test_formats/bls/README.md index 700686bdd7..db63bba1d3 100644 --- a/specs/test_formats/bls/README.md +++ b/specs/test_formats/bls/README.md @@ -1,6 +1,6 @@ -# BLS test suite +# BLS tests -A test suite for BLS. Primarily geared towards verifying the *integration* of any BLS library. +A test type for BLS. Primarily geared towards verifying the *integration* of any BLS library. We do not recommend to roll your own crypto, or use an untested BLS library. The BLS test suite runner has the following handlers: diff --git a/specs/test_formats/operations/README.md b/specs/test_formats/operations/README.md index 747d8217c4..842dc3615f 100644 --- a/specs/test_formats/operations/README.md +++ b/specs/test_formats/operations/README.md @@ -1,4 +1,10 @@ -# Operations test suite +# Operations tests + +The different kinds of operations ("transactions") are tested individually with test handlers. + +The tested operation kinds are: +- [`deposits`](./deposits.md) +- More tests are work-in-progress. + -The operations test suite From 0aa1cc8f6094b1e7f6ca6d0391bcbce3575950e7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 22:30:19 +1000 Subject: [PATCH 36/61] update shuffling test docs --- specs/test_formats/shuffling/README.md | 36 ++++++++++++++++++++++++++ test_generators/shuffling/README.md | 6 ----- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/specs/test_formats/shuffling/README.md b/specs/test_formats/shuffling/README.md index e69de29bb2..efc1b7b1ae 100644 --- a/specs/test_formats/shuffling/README.md +++ b/specs/test_formats/shuffling/README.md @@ -0,0 +1,36 @@ +# Test format: shuffling + +The runner of the Shuffling test type has only one handler: `core` + +This does not mean however that testing is limited. +Clients may take different approaches to shuffling, for optimizing, + and supporting advanced lookup behavior back in older history. + +For implementers, possible test runners implementing testing can include: +1) just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation) +2) test un-permute-index (the reverse lookup. Implemented by running the shuffling rounds in reverse: from `round_count-1` to `0`) +3) test the optimized complete shuffle, where all indices are shuffled at once, test output in one go. +4) test complete shuffle in reverse (reverse rounds, same as 2) + +## Test case format + +```yaml +seed: bytes32 +count: int +shuffled: List[int] +``` + +- The `bytes32` is encoded as strings, hexadecimal encoding, prefixed with `0x`. +- Integers are validator indices. These are `uint64`, but realistically they are not as big. + +The `count` specifies the validator registry size. One should compute the shuffling for indices `0, 1, 2, 3, ..., count (exclusive)`. +Seed is the raw shuffling seed, passed to permute-index (or optimized shuffling approach). + +## Condition + +The resulting list should match the expected output `shuffled` after shuffling the implied input, using the given `seed`. + +## Forks + +Forks-interpretation: `collective` + diff --git a/test_generators/shuffling/README.md b/test_generators/shuffling/README.md index 5cf9d50e06..a8f0cbdb45 100644 --- a/test_generators/shuffling/README.md +++ b/test_generators/shuffling/README.md @@ -2,12 +2,6 @@ Tests for the swap-or-not shuffling in ETH 2.0. -For implementers, possible test runners implementing testing can include: -1) just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation) -2) test un-permute-index (the reverse lookup. Implemented by running the shuffling rounds in reverse: from `round_count-1` to `0`) -3) test the optimized complete shuffle, where all indices are shuffled at once, test output in one go. -4) test complete shuffle in reverse (reverse rounds, same as 2) - Tips for initial shuffling write: - run with `round_count = 1` first, do the same with pyspec. - start with permute index From 88d076b615d5b45887faece7dbdfaa968b7e8f7a Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 22:48:59 +1000 Subject: [PATCH 37/61] initial work to fix outdated ssz tests --- specs/test_formats/ssz/uint.md | 26 ++++++++++++++++++++++++++ test_generators/ssz/main.py | 6 +++--- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 specs/test_formats/ssz/uint.md diff --git a/specs/test_formats/ssz/uint.md b/specs/test_formats/ssz/uint.md new file mode 100644 index 0000000000..c05a343af9 --- /dev/null +++ b/specs/test_formats/ssz/uint.md @@ -0,0 +1,26 @@ +# Test format: SSZ uints + +SSZ supports encoding of uints up to 32 bytes. These are considered to be basic types. + +## Test case format + +```yaml +TODO: old format +# type: "uintN" -- string, where N is one of [8, 16, 32, 64, 128, 256] +# valid: bool -- expected validity of the input data +# ssz: bytes -- string, input data, hex encoded, with prefix 0x +# tags: List[string] -- description of test case, in the form of a list of labels +``` + +## Condition + +- Encoding: After encoding the given input number, the +- Decoding: After decoding the given `output` bytes, it should match the `input` number + +## Forks + +Forks-interpretation: `collective` + +``` + +``` \ No newline at end of file diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py index f6454c3c09..e53204df01 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz/main.py @@ -8,7 +8,7 @@ from gen_base import gen_runner, gen_suite, gen_typing def ssz_random_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("uint_random", "core", gen_suite.render_suite( + return ("uint_random", "uint", gen_suite.render_suite( title="UInt Random", summary="Random integers chosen uniformly over the allowed value range", forks_timeline= "mainnet", @@ -18,7 +18,7 @@ def ssz_random_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: test_cases=generate_random_uint_test_cases())) def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("uint_wrong_length", "core", gen_suite.render_suite( + return ("uint_wrong_length", "uint", gen_suite.render_suite( title="UInt Wrong Length", summary="Serialized integers that are too short or too long", forks_timeline= "mainnet", @@ -28,7 +28,7 @@ def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: test_cases=generate_uint_wrong_length_test_cases())) def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: - return ("uint_bounds", "core", gen_suite.render_suite( + return ("uint_bounds", "uint", gen_suite.render_suite( title="UInt Bounds", summary="Integers right at or beyond the bounds of the allowed value range", forks_timeline= "mainnet", From b6c45b9dc1326761b88aa05811416a0fc885530a Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 23:02:36 +1000 Subject: [PATCH 38/61] update ssz test format and fix uint sizes --- specs/test_formats/ssz/uint.md | 19 ++++++++----------- test_generators/ssz/main.py | 2 +- test_generators/ssz/uint_test_cases.py | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/specs/test_formats/ssz/uint.md b/specs/test_formats/ssz/uint.md index c05a343af9..f71ddecb88 100644 --- a/specs/test_formats/ssz/uint.md +++ b/specs/test_formats/ssz/uint.md @@ -5,22 +5,19 @@ SSZ supports encoding of uints up to 32 bytes. These are considered to be basic ## Test case format ```yaml -TODO: old format -# type: "uintN" -- string, where N is one of [8, 16, 32, 64, 128, 256] -# valid: bool -- expected validity of the input data -# ssz: bytes -- string, input data, hex encoded, with prefix 0x -# tags: List[string] -- description of test case, in the form of a list of labels +type: "uintN" -- string, where N is one of [8, 16, 32, 64, 128, 256] +valid: bool -- expected validity of the input data +value: string -- string, decimal encoding, to support up to 256 bit integers +ssz: bytes -- string, input data, hex encoded, with prefix 0x +tags: List[string] -- description of test case, in the form of a list of labels ``` ## Condition -- Encoding: After encoding the given input number, the -- Decoding: After decoding the given `output` bytes, it should match the `input` number +Two-way testing can be implemented in the test-runner: +- Encoding: After encoding the given input number `value`, the output should match `ssz` +- Decoding: After decoding the given `ssz` bytes, it should match the input number `value` ## Forks Forks-interpretation: `collective` - -``` - -``` \ No newline at end of file diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py index e53204df01..0e5d9d7c8c 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz/main.py @@ -39,4 +39,4 @@ def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: if __name__ == "__main__": - gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) \ No newline at end of file + gen_runner.run_generator("ssz", [ssz_random_uint_suite, ssz_wrong_uint_suite, ssz_uint_bounds_suite]) diff --git a/test_generators/ssz/uint_test_cases.py b/test_generators/ssz/uint_test_cases.py index d123564ca1..9ede168480 100644 --- a/test_generators/ssz/uint_test_cases.py +++ b/test_generators/ssz/uint_test_cases.py @@ -15,7 +15,7 @@ random.seed(0) -BIT_SIZES = [i for i in range(8, 512 + 1, 8)] +BIT_SIZES = [8, 16, 32, 64, 128, 256] RANDOM_TEST_CASES_PER_BIT_SIZE = 10 RANDOM_TEST_CASES_PER_LENGTH = 3 From 6b701a6c8b0dedb097163902c19d07d68f024cdd Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Apr 2019 23:59:19 +1000 Subject: [PATCH 39/61] update tests format docs --- specs/test_formats/README.md | 26 +++++++++++++++++++------- specs/test_formats/ssz/README.md | 15 +++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 71e72f0cb9..53973495da 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -17,8 +17,17 @@ This document defines the YAML format and structure used for ETH 2.0 testing. Ethereum 2.0 uses YAML as the format for all cross client tests. This document describes at a high level the general format to which all test files should conform. +### Test-case formats + The particular formats of specific types of tests (test suites) are defined in separate documents. +Test formats: +- [`bls`](./bls/README.md) +- [`operations`](./operations/README.md) +- [`shuffling`](./shuffling/README.md) +- [`ssz`](./ssz/README.md) +- More formats are planned, see tracking issues for CI/testing + ## Glossary - `generator`: a program that outputs one or more `suite` files. @@ -167,11 +176,14 @@ To prevent parsing of hundreds of different YAML files to test a specific test t or even more specific, just a handler, tests should be structured in the following nested form: ``` -. <--- root of eth2.0 tests repository -├── bls <--- collection of handler for a specific test-runner, example runner: "bls" -│   ├── signing <--- collection of test suites for a specific handler, example handler: "signing". If no multiple handlers, use a dummy folder (e.g. "main"), and specify that in the yaml. -│   │   ├── sign_msg.yml <--- an entry list of test suites -│   │   ... <--- more suite files (optional) -│   ... <--- more handlers -... <--- more test types +. <--- root of eth2.0 tests repository +├── bls <--- collection of handler for a specific test-runner, example runner: "bls" +│   ├── verify_msg <--- collection of test suites for a specific handler, example handler: "verify_msg". If no multiple handlers, use a dummy folder (e.g. "core"), and specify that in the yaml. +│   │   ├── verify_valid.yml . +│   │   ├── special_cases.yml . a list of test suites +│   │   ├── domains.yml . +│   │   ├── invalid.yml . +│   │   ... <--- more suite files (optional) +│   ... <--- more handlers +... <--- more test types ``` diff --git a/specs/test_formats/ssz/README.md b/specs/test_formats/ssz/README.md index e69de29bb2..72ba7dac10 100644 --- a/specs/test_formats/ssz/README.md +++ b/specs/test_formats/ssz/README.md @@ -0,0 +1,15 @@ +# SSZ tests + +SSZ has changed throughout the development of ETH 2.0. + +## Contents + +A minimal but useful series of tests covering `uint` encoding and decoding is provided. +This is a direct port of the older SSZ `uint` tests (minus outdated test cases). + +[uint test format](./uint.md). + +Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. +The exact uint lengths to support may be redefined in the future. + +Extension of the SSZ tests collection is planned, see CI/testing issues for progress tracking. From fcc4dc3710557da1837cb2c4e53b78fcc95857e1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 13 Apr 2019 00:03:04 +1000 Subject: [PATCH 40/61] temporarily disable the yaml save and commit, exact workflow TBD in future PR --- .circleci/config.yml | 133 ++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c347a064f4..d40fd467f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,68 +19,71 @@ jobs: name: Generate YAML tests command: make gen_yaml_tests - - store_artifacts: - path: test-reports - destination: test-reports - - - run: - name: Save YAML tests for deployment - command: | - mkdir /tmp/workspace - cp -r yaml_tests /tmp/workspace/ - git log -1 >> /tmp/workspace/latest_commit_message - - persist_to_workspace: - root: /tmp/workspace - paths: - - yaml_tests - - latest_commit_message - commit: - docker: - - image: circleci/python:3.6 - steps: - - attach_workspace: - at: /tmp/workspace - - add_ssh_keys: - fingerprints: - - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa" - - run: - name: Checkout test repository - command: | - ssh-keyscan -H github.com >> ~/.ssh/known_hosts - git clone git@github.com:ethereum/eth2.0-tests.git - - run: - name: Commit and push generated YAML tests - command: | - cd eth2.0-tests - git config user.name 'eth2TestGenBot' - git config user.email '47188154+eth2TestGenBot@users.noreply.github.com' - for filename in /tmp/workspace/yaml_tests/*; do - rm -rf $(basename $filename) - cp -r $filename . - done - git add . - if git diff --cached --exit-code >& /dev/null; then - echo "No changes to commit" - else - echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message - cat /tmp/workspace/latest_commit_message >> commit_message - git commit -F commit_message - git push origin master - fi -workflows: - version: 2.1 - - build_and_commit: - jobs: - - build: - filters: - tags: - only: /.*/ - - commit: - requires: - - build - filters: - tags: - only: /.*/ - branches: - ignore: /.*/ \ No newline at end of file +# TODO in future PR (after #851): decide on CI triggering of yaml tests building, +# and destination of output (new yaml tests LFS-configured repository) +# +# - store_artifacts: +# path: test-reports +# destination: test-reports +# +# - run: +# name: Save YAML tests for deployment +# command: | +# mkdir /tmp/workspace +# cp -r yaml_tests /tmp/workspace/ +# git log -1 >> /tmp/workspace/latest_commit_message +# - persist_to_workspace: +# root: /tmp/workspace +# paths: +# - yaml_tests +# - latest_commit_message +# commit: +# docker: +# - image: circleci/python:3.6 +# steps: +# - attach_workspace: +# at: /tmp/workspace +# - add_ssh_keys: +# fingerprints: +# - "01:85:b6:36:96:a6:84:72:e4:9b:4e:38:ee:21:97:fa" +# - run: +# name: Checkout test repository +# command: | +# ssh-keyscan -H github.com >> ~/.ssh/known_hosts +# git clone git@github.com:ethereum/eth2.0-tests.git +# - run: +# name: Commit and push generated YAML tests +# command: | +# cd eth2.0-tests +# git config user.name 'eth2TestGenBot' +# git config user.email '47188154+eth2TestGenBot@users.noreply.github.com' +# for filename in /tmp/workspace/yaml_tests/*; do +# rm -rf $(basename $filename) +# cp -r $filename . +# done +# git add . +# if git diff --cached --exit-code >& /dev/null; then +# echo "No changes to commit" +# else +# echo -e "Update generated tests\n\nLatest commit message from eth2.0-specs:\n" > commit_message +# cat /tmp/workspace/latest_commit_message >> commit_message +# git commit -F commit_message +# git push origin master +# fi +#workflows: +# version: 2.1 +# +# build_and_commit: +# jobs: +# - build: +# filters: +# tags: +# only: /.*/ +# - commit: +# requires: +# - build +# filters: +# tags: +# only: /.*/ +# branches: +# ignore: /.*/ \ No newline at end of file From 18d54fa1f82aba3270e3ada8133d7746f64893e0 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 14 Apr 2019 19:38:37 +1000 Subject: [PATCH 41/61] fix deposit contract placeholder address length --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 2 +- scripts/phase0/function_puller.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 27085d40ab..d2d91d02cf 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -22,7 +22,7 @@ SHUFFLE_ROUND_COUNT: 90 # Deposit contract # --------------------------------------------------------------- # **TBD** -DEPOSIT_CONTRACT_ADDRESS: 0x12345678901235678901234567890123567890 +DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 # 2**5 ` (= 32) DEPOSIT_CONTRACT_TREE_DEPTH: 32 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 8997bc5ed2..c1d69bcd40 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -22,7 +22,7 @@ SHUFFLE_ROUND_COUNT: 10 # Deposit contract # --------------------------------------------------------------- # **TBD** -DEPOSIT_CONTRACT_ADDRESS: 0x12345678901235678901234567890123567890 +DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 # 2**5 ` (= 32) DEPOSIT_CONTRACT_TREE_DEPTH: 32 diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index fc7f9fb8c1..812498b2bd 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -53,7 +53,7 @@ def get_spec(file_name) -> List[str]: if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': eligible = False if eligible: - code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123567890123456789012357890'))) + code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) # Build type-def re-initialization code_lines.append('') code_lines.append('def init_SSZ_types():') From 514d8f9232a58c8117a51c271405bf95dc6a4695 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 14 Apr 2019 20:17:22 +1000 Subject: [PATCH 42/61] Fix runner/handler references --- test_generators/bls/main.py | 6 ++++++ test_generators/operations/deposits.py | 6 ++++-- test_generators/shuffling/main.py | 2 ++ test_generators/ssz/main.py | 9 ++++++--- test_libs/gen_helpers/gen_base/gen_suite.py | 2 ++ 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index da6a79aaed..ef80635dee 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -166,6 +166,7 @@ def bls_msg_hash_uncompressed_suite(configs_path: str) -> gen_typing.TestSuiteOu forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="bls", handler="msg_hash_uncompressed", test_cases=case01_message_hash_G2_uncompressed())) @@ -177,6 +178,7 @@ def bls_msg_hash_compressed_suite(configs_path: str) -> gen_typing.TestSuiteOutp forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="bls", handler="msg_hash_compressed", test_cases=case02_message_hash_G2_compressed())) @@ -189,6 +191,7 @@ def bls_priv_to_pub_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="bls", handler="priv_to_pub", test_cases=case03_private_to_public_key())) @@ -200,6 +203,7 @@ def bls_sign_msg_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="bls", handler="sign_msg", test_cases=case04_sign_messages())) @@ -211,6 +215,7 @@ def bls_aggregate_sigs_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="bls", handler="aggregate_sigs", test_cases=case06_aggregate_sigs())) @@ -222,6 +227,7 @@ def bls_aggregate_pubkeys_suite(configs_path: str) -> gen_typing.TestSuiteOutput forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="bls", handler="aggregate_pubkeys", test_cases=case07_aggregate_pubkeys())) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index a72b1fbaa6..e92731cbb8 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -155,7 +155,8 @@ def mini_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="testing", forks=["phase0"], config="minimal", - handler="core", + runner="operations", + handler="deposits", test_cases=deposit_cases())) @@ -169,5 +170,6 @@ def full_deposits_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="mainnet", forks=["phase0"], config="mainnet", - handler="core", + runner="operations", + handler="deposits", test_cases=deposit_cases())) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index e8b2054a2a..2c4faeb8fb 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -30,6 +30,7 @@ def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="testing", forks=["phase0"], config="minimal", + runner="shuffling", handler="core", test_cases=shuffling_test_cases())) @@ -44,6 +45,7 @@ def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline="mainnet", forks=["phase0"], config="mainnet", + runner="shuffling", handler="core", test_cases=shuffling_test_cases())) diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py index 0e5d9d7c8c..c1af4ce5f8 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz/main.py @@ -14,7 +14,8 @@ def ssz_random_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline= "mainnet", forks=["phase0"], config="mainnet", - handler="core", + runner="ssz", + handler="uint", test_cases=generate_random_uint_test_cases())) def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: @@ -24,7 +25,8 @@ def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline= "mainnet", forks=["phase0"], config="mainnet", - handler="core", + runner="ssz", + handler="uint", test_cases=generate_uint_wrong_length_test_cases())) def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: @@ -34,7 +36,8 @@ def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: forks_timeline= "mainnet", forks=["phase0"], config="mainnet", - handler="core", + runner="ssz", + handler="uint", test_cases=generate_uint_bounds_test_cases() + generate_uint_out_of_bounds_test_cases())) diff --git a/test_libs/gen_helpers/gen_base/gen_suite.py b/test_libs/gen_helpers/gen_base/gen_suite.py index 3459d9ae35..a3f88791fb 100644 --- a/test_libs/gen_helpers/gen_base/gen_suite.py +++ b/test_libs/gen_helpers/gen_base/gen_suite.py @@ -9,6 +9,7 @@ def render_suite(*, title: str, summary: str, forks_timeline: str, forks: Iterable[str], config: str, + runner: str, handler: str, test_cases: Iterable[TestCase]): yield "title", title @@ -16,5 +17,6 @@ def render_suite(*, yield "forks_timeline", forks_timeline, yield "forks", forks yield "config", config + yield "runner", runner yield "handler", handler yield "test_cases", test_cases From 9bb902217d5ba66061e589ac22d41df66750eb4d Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 14 Apr 2019 22:54:01 +1000 Subject: [PATCH 43/61] change wording deposit case format --- specs/test_formats/operations/deposits.md | 8 ++++---- test_generators/operations/deposits.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/test_formats/operations/deposits.md b/specs/test_formats/operations/deposits.md index 5aaed24f79..2bce84e30b 100644 --- a/specs/test_formats/operations/deposits.md +++ b/specs/test_formats/operations/deposits.md @@ -5,10 +5,10 @@ A deposit is a form of an operation (or "transaction"), modifying the state. ## Test case format ```yaml -case: string -- description of test case, purely for debugging purposes -pre: BeaconState -- state before applying the deposit -deposit: Deposit -- the deposit -post: BeaconState -- state after applying the deposit. No value if deposit processing is aborted. +description: string -- description of test case, purely for debugging purposes +pre: BeaconState -- state before applying the deposit +deposit: Deposit -- the deposit +post: BeaconState -- state after applying the deposit. No value if deposit processing is aborted. ``` ## Condition diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index e92731cbb8..bc2b842156 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -91,7 +91,7 @@ def valid_deposit(): new_dep, state, leaves = build_deposit_for_index(10, 10) state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) state.latest_eth1_data.deposit_count = len(leaves) - yield 'case', 'valid deposit to add new validator' + yield 'description', 'valid deposit to add new validator' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) spec.process_deposit(state, new_dep) @@ -103,7 +103,7 @@ def valid_topup(): new_dep, state, leaves = build_deposit_for_index(10, 3) state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) state.latest_eth1_data.deposit_count = len(leaves) - yield 'case', 'valid deposit to top-up existing validator' + yield 'description', 'valid deposit to top-up existing validator' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) spec.process_deposit(state, new_dep) @@ -118,7 +118,7 @@ def invalid_deposit_index(): # Mess up deposit index, 1 too small state.deposit_index = 9 - yield 'case', 'invalid deposit index' + yield 'description', 'invalid deposit index' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) yield 'post', None @@ -131,7 +131,7 @@ def invalid_deposit_proof(): # Make deposit proof invalid (at bottom of proof) new_dep.proof[-1] = spec.ZERO_HASH - yield 'case', 'invalid deposit proof' + yield 'description', 'invalid deposit proof' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) yield 'post', None From 9591bea42c61e9619a845ce27511a1452630a9cd Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Apr 2019 16:01:17 +1000 Subject: [PATCH 44/61] cleanup deposit tests --- test_generators/operations/deposits.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index bc2b842156..b75025f0b0 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -58,11 +58,12 @@ def build_deposit(state, index=index, data=deposit_data, ) + assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, get_merkle_root(tuple(deposit_data_leaves))) return deposit -def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[spec.Deposit, spec.BeaconState, List[spec.Bytes32]]: +def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[spec.Deposit, spec.BeaconState]: genesis_deposits = genesis.create_deposits( keys.pubkeys[:initial_validator_count], keys.withdrawal_creds[:initial_validator_count] @@ -83,14 +84,12 @@ def build_deposit_for_index(initial_validator_count: int, index: int) -> Tuple[s state.latest_eth1_data.deposit_root = get_merkle_root(tuple(deposit_data_leaves)) state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - return deposit, state, deposit_data_leaves + return deposit, state @to_dict def valid_deposit(): - new_dep, state, leaves = build_deposit_for_index(10, 10) - state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) - state.latest_eth1_data.deposit_count = len(leaves) + new_dep, state = build_deposit_for_index(10, 10) yield 'description', 'valid deposit to add new validator' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) @@ -100,9 +99,7 @@ def valid_deposit(): @to_dict def valid_topup(): - new_dep, state, leaves = build_deposit_for_index(10, 3) - state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) - state.latest_eth1_data.deposit_count = len(leaves) + new_dep, state = build_deposit_for_index(10, 3) yield 'description', 'valid deposit to top-up existing validator' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) @@ -112,9 +109,7 @@ def valid_topup(): @to_dict def invalid_deposit_index(): - new_dep, state, leaves = build_deposit_for_index(10, 10) - state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) - state.latest_eth1_data.deposit_count = len(leaves) + new_dep, state = build_deposit_for_index(10, 10) # Mess up deposit index, 1 too small state.deposit_index = 9 @@ -125,9 +120,7 @@ def invalid_deposit_index(): @to_dict def invalid_deposit_proof(): - new_dep, state, leaves = build_deposit_for_index(10, 10) - state.latest_eth1_data.deposit_root = get_merkle_root(tuple(leaves)) - state.latest_eth1_data.deposit_count = len(leaves) + new_dep, state = build_deposit_for_index(10, 10) # Make deposit proof invalid (at bottom of proof) new_dep.proof[-1] = spec.ZERO_HASH From 25dd3b8d8e93f8a8f3dc6a06ef34fadce37b31c6 Mon Sep 17 00:00:00 2001 From: Dmitry S Date: Mon, 15 Apr 2019 22:29:25 +1000 Subject: [PATCH 45/61] Update test_generators/README.md Co-Authored-By: protolambda --- test_generators/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/README.md b/test_generators/README.md index bd509904c1..1393af0682 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -1,6 +1,6 @@ # Eth2.0 Test Generators -This directory of contains all the generators for YAML tests, consumed by Eth 2.0 client implementations. +This directory contains all the generators for YAML tests, consumed by Eth 2.0 client implementations. Any issues with the generators and/or generated tests should be filed in the repository that hosts the generator outputs, here: [ethereum/eth2.0-tests](https://github.com/ethereum/eth2.0-tests/). From 0b2a03e2768a4f7ff30161ae5c3721e2abe5941a Mon Sep 17 00:00:00 2001 From: Dmitry S Date: Mon, 15 Apr 2019 22:30:02 +1000 Subject: [PATCH 46/61] Update test_generators/README.md Co-Authored-By: protolambda --- test_generators/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/README.md b/test_generators/README.md index 1393af0682..8a34cb51ec 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -132,7 +132,7 @@ if __name__ == "__main__": Recommendations: - you can have more than just 1 suite creator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])` -- you can concatenate lists of test cases, if you don't want to split it up in suites. +- you can concatenate lists of test cases, if you don't want to split it up in suites, however make sure they could be run with one handler. - you can split your suite creators into different python files/packages, good for code organization. - use config "minimal" for performance. But also implement a suite with the default config where necessary. - you may be able to write your test suite creator in a way where it does not make assumptions on constants. From d64a4f248eb38f407fcfb2654a7adceefb9a6f4f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Apr 2019 22:39:07 +1000 Subject: [PATCH 47/61] forks coverage description cleanup --- specs/test_formats/README.md | 24 +++++-------------- specs/test_formats/bls/aggregate_pubkeys.md | 4 ---- specs/test_formats/bls/aggregate_sigs.md | 4 ---- .../bls/msg_hash_g2_compressed.md | 4 ---- .../bls/msg_hash_g2_uncompressed.md | 4 ---- specs/test_formats/bls/priv_to_pub.md | 4 ---- specs/test_formats/bls/sign_msg.md | 4 ---- specs/test_formats/operations/deposits.md | 8 ------- specs/test_formats/shuffling/README.md | 4 ---- specs/test_formats/ssz/uint.md | 4 ---- 10 files changed, 6 insertions(+), 58 deletions(-) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 18c7329d9a..6b9533056c 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -75,22 +75,11 @@ There are two types of fork-data: The first is neat to have as a separate form: we prevent duplication, and can run with different presets (e.g. fork timeline for a minimal local test, for a public testnet, or for mainnet) -The second is still somewhat ambiguous: some tests may want cover multiple forks, and can do so in different ways: -- run one test, transitioning from one to the other -- run the same test for both -- run a test for every transition from one fork to the other -- more - -There is a common factor here however: the options are exclusive, and give a clear idea on what test suites need to be ran to cover testing for a specific fork. -The way this list of forks is interpreted, is up to the test-runner: -State-transition test suites may want to just declare forks that are being covered in the test suite, - whereas shuffling test suites may want to declare a list of forks to test the shuffling algorithm for individually. - -Test-formats specify the following `forks` interpretation rules: - -- `collective`: the test suite applies to all specified forks, and only needs to run once -- `individual`: the test suite should be ran against every fork -- more types may be specified with future test types. +The second does not affect the result of the tests, it just states what is covered by the tests, + so that the right suites can be executed to see coverage for a certain fork. +For some types of tests, it may be beneficial to ensure it runs exactly the same, with any given fork "active". +Test-formats can be explicit on the need to repeat a test with different forks being "active", + but generally tests run only once. ### Test completeness @@ -107,8 +96,7 @@ The aim is to provide clients with a well-defined scope of work to run a particu title: -- Display name for the test suite summary: -- Summarizes the test suite forks_timeline: -- Used to determine the forking timeline -forks: -- Runner decides what to do: run for each fork, or run for all at once, each fork transition, etc. - - ... +forks: -- Defines the coverage. Test-runner code may decide to re-run with the different forks "activated", when applicable. config: -- Used to determine which set of constants to run (possibly compile time) with runner: *MUST be consistent with folder structure* handler: *MUST be consistent with folder structure* diff --git a/specs/test_formats/bls/aggregate_pubkeys.md b/specs/test_formats/bls/aggregate_pubkeys.md index 9a6f1cc256..43c7d6c6dd 100644 --- a/specs/test_formats/bls/aggregate_pubkeys.md +++ b/specs/test_formats/bls/aggregate_pubkeys.md @@ -15,7 +15,3 @@ output: BLS Pubkey -- expected output, single BLS pubkey ## Condition The `aggregate_pubkeys` handler should aggregate the keys in the `input`, and the result should match the expected `output`. - -## Forks - -Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/aggregate_sigs.md b/specs/test_formats/bls/aggregate_sigs.md index 1588e26cba..6690c3344f 100644 --- a/specs/test_formats/bls/aggregate_sigs.md +++ b/specs/test_formats/bls/aggregate_sigs.md @@ -15,7 +15,3 @@ output: BLS Signature -- expected output, single BLS signature ## Condition The `aggregate_sigs` handler should aggregate the signatures in the `input`, and the result should match the expected `output`. - -## Forks - -Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/msg_hash_g2_compressed.md b/specs/test_formats/bls/msg_hash_g2_compressed.md index 51c64e28b4..4e194e90bc 100644 --- a/specs/test_formats/bls/msg_hash_g2_compressed.md +++ b/specs/test_formats/bls/msg_hash_g2_compressed.md @@ -17,7 +17,3 @@ All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with ` ## Condition The `msg_hash_g2_compressed` handler should hash the `message`, with the given `domain`, to G2 with compression, and the result should match the expected `output`. - -## Forks - -Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/msg_hash_g2_uncompressed.md b/specs/test_formats/bls/msg_hash_g2_uncompressed.md index b7d2caa02b..f42ea9998d 100644 --- a/specs/test_formats/bls/msg_hash_g2_uncompressed.md +++ b/specs/test_formats/bls/msg_hash_g2_uncompressed.md @@ -17,7 +17,3 @@ All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with ` ## Condition The `msg_hash_g2_uncompressed` handler should hash the `message`, with the given `domain`, to G2, without compression, and the result should match the expected `output`. - -## Forks - -Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/priv_to_pub.md b/specs/test_formats/bls/priv_to_pub.md index 9265b83ed3..7af148d0f8 100644 --- a/specs/test_formats/bls/priv_to_pub.md +++ b/specs/test_formats/bls/priv_to_pub.md @@ -15,7 +15,3 @@ All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with ` ## Condition The `priv_to_pub` handler should compute the public key for the given private key `input`, and the result should match the expected `output`. - -## Forks - -Forks-interpretation: `collective` diff --git a/specs/test_formats/bls/sign_msg.md b/specs/test_formats/bls/sign_msg.md index 3a6d63fa27..dd93174f29 100644 --- a/specs/test_formats/bls/sign_msg.md +++ b/specs/test_formats/bls/sign_msg.md @@ -18,7 +18,3 @@ All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with ` ## Condition The `sign_msg` handler should sign the given `message`, with `domain`, using the given `privkey`, and the result should match the expected `output`. - -## Forks - -Forks-interpretation: `collective` diff --git a/specs/test_formats/operations/deposits.md b/specs/test_formats/operations/deposits.md index 2bce84e30b..b9dce318a0 100644 --- a/specs/test_formats/operations/deposits.md +++ b/specs/test_formats/operations/deposits.md @@ -16,11 +16,3 @@ post: BeaconState -- state after applying the deposit. No value if deposit pr A `deposits` handler of the `operations` should process these cases, calling the implementation of the `process_deposit(state, deposit)` functionality described in the spec. The resulting state should match the expected `post` state, or no change if the `post` state is left blank. - -## Forks - -Forks-interpretation: `collective` - -Pre and post state contain slot numbers, and are time sensitive. -Additional tests will be added for future forks to cover fork-specific behavior based on input data - (including suites with deposits on fork transition blocks, covering multiple forks) diff --git a/specs/test_formats/shuffling/README.md b/specs/test_formats/shuffling/README.md index efc1b7b1ae..514baf15aa 100644 --- a/specs/test_formats/shuffling/README.md +++ b/specs/test_formats/shuffling/README.md @@ -30,7 +30,3 @@ Seed is the raw shuffling seed, passed to permute-index (or optimized shuffling The resulting list should match the expected output `shuffled` after shuffling the implied input, using the given `seed`. -## Forks - -Forks-interpretation: `collective` - diff --git a/specs/test_formats/ssz/uint.md b/specs/test_formats/ssz/uint.md index f71ddecb88..fd7cf3221e 100644 --- a/specs/test_formats/ssz/uint.md +++ b/specs/test_formats/ssz/uint.md @@ -17,7 +17,3 @@ tags: List[string] -- description of test case, in the form of a list of labels Two-way testing can be implemented in the test-runner: - Encoding: After encoding the given input number `value`, the output should match `ssz` - Decoding: After decoding the given `ssz` bytes, it should match the input number `value` - -## Forks - -Forks-interpretation: `collective` From 87bee73222604a9abc40c3ead8f12c5c50f5e34c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:05:42 +1000 Subject: [PATCH 48/61] Update specs/test_formats/operations/deposits.md Co-Authored-By: protolambda --- specs/test_formats/operations/deposits.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/operations/deposits.md b/specs/test_formats/operations/deposits.md index b9dce318a0..8f44ebb228 100644 --- a/specs/test_formats/operations/deposits.md +++ b/specs/test_formats/operations/deposits.md @@ -15,4 +15,4 @@ post: BeaconState -- state after applying the deposit. No value if deposit pr A `deposits` handler of the `operations` should process these cases, calling the implementation of the `process_deposit(state, deposit)` functionality described in the spec. -The resulting state should match the expected `post` state, or no change if the `post` state is left blank. +The resulting state should match the expected `post` state, or if the `post` state is left blank, the handler should reject the inputs as invalid. From 956e7c5abc8e8b28025b8be46fb2ee41eb995bb8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:06:01 +1000 Subject: [PATCH 49/61] Update specs/test_formats/shuffling/README.md Co-Authored-By: protolambda --- specs/test_formats/shuffling/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/test_formats/shuffling/README.md b/specs/test_formats/shuffling/README.md index 514baf15aa..57be965657 100644 --- a/specs/test_formats/shuffling/README.md +++ b/specs/test_formats/shuffling/README.md @@ -20,7 +20,7 @@ count: int shuffled: List[int] ``` -- The `bytes32` is encoded as strings, hexadecimal encoding, prefixed with `0x`. +- The `bytes32` is encoded a string, hexadecimal encoding, prefixed with `0x`. - Integers are validator indices. These are `uint64`, but realistically they are not as big. The `count` specifies the validator registry size. One should compute the shuffling for indices `0, 1, 2, 3, ..., count (exclusive)`. From 3b9d35f8e8ce5f8ca096a22dd8ae1f43f9881028 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:07:42 +1000 Subject: [PATCH 50/61] Update test_generators/README.md Co-Authored-By: protolambda --- test_generators/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/README.md b/test_generators/README.md index 8a34cb51ec..743157aae4 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -63,7 +63,7 @@ eth-utils==1.4.1 ../../test_libs/pyspec ``` The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself, to prevent code duplication and outdated tests. -Applying configurations to the spec is easy, and enables you to create test suites with different contexts. +Applying configurations to the spec is simple, and enables you to create test suites with different contexts. Note: make sure to run `make pyspec` from the root of the specs repository, to build the pyspec requirement. From 8a1aa275a3a197b21523f177830b1e74968d76ee Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:08:05 +1000 Subject: [PATCH 51/61] Update test_generators/operations/README.md Co-Authored-By: protolambda --- test_generators/operations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/README.md b/test_generators/operations/README.md index 9f1ecfddb8..4e46c8dcbc 100644 --- a/test_generators/operations/README.md +++ b/test_generators/operations/README.md @@ -3,7 +3,7 @@ Operations (or "transactions" in previous spec iterations), are atomic changes to the state, introduced by embedding in blocks. -This generators provides a series of test suites, divided into handler, for each operation type. +This generator provides a series of test suites, divided into handler, for each operation type. A operation test-runner can consume these operation test-suites, and handle different kinds of operations by processing the cases using the specified test handler. From 0400a888a69c0105a2a142b0912b54522636b2e7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:08:27 +1000 Subject: [PATCH 52/61] Update test_generators/operations/README.md Co-Authored-By: protolambda --- test_generators/operations/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/operations/README.md b/test_generators/operations/README.md index 4e46c8dcbc..e0b9d0e187 100644 --- a/test_generators/operations/README.md +++ b/test_generators/operations/README.md @@ -4,7 +4,7 @@ Operations (or "transactions" in previous spec iterations), are atomic changes to the state, introduced by embedding in blocks. This generator provides a series of test suites, divided into handler, for each operation type. -A operation test-runner can consume these operation test-suites, +An operation test-runner can consume these operation test-suites, and handle different kinds of operations by processing the cases using the specified test handler. Information on the format of the tests can be found in the [operations test formats documentation](../../specs/test_formats/operations/README.md). From 6542ae2c4ddcf5d07ced127d1717e7cf96beec63 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:08:49 +1000 Subject: [PATCH 53/61] Update test_generators/operations/deposits.py Co-Authored-By: protolambda --- test_generators/operations/deposits.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index b75025f0b0..3e47482a66 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -118,6 +118,7 @@ def invalid_deposit_index(): yield 'deposit', encode(new_dep, spec.Deposit) yield 'post', None + @to_dict def invalid_deposit_proof(): new_dep, state = build_deposit_for_index(10, 10) From 79c8f562bd10307ddb11bb275b34e3a60ae5750d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:09:05 +1000 Subject: [PATCH 54/61] Update test_generators/ssz/renderers.py Co-Authored-By: protolambda --- test_generators/ssz/renderers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz/renderers.py b/test_generators/ssz/renderers.py index ee8a928387..28571cddaf 100644 --- a/test_generators/ssz/renderers.py +++ b/test_generators/ssz/renderers.py @@ -77,7 +77,7 @@ def render_test_case(*, sedes, valid, value=None, serial=None, description=None, raise ValueError("For valid test cases, both value and ssz must be present") else: if value_and_serial_given: - raise ValueError("For invalid test cases, either value or ssz must not be present") + raise ValueError("For invalid test cases, one of either value or ssz must not be present") if tags is None: tags = [] From 943989c611b2b8353a0db735c6b321f6b68c6b3b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:31:10 +1000 Subject: [PATCH 55/61] Update test_generators/ssz/main.py Co-Authored-By: protolambda --- test_generators/ssz/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py index c1af4ce5f8..e1c67cd9ed 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz/main.py @@ -18,6 +18,7 @@ def ssz_random_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: handler="uint", test_cases=generate_random_uint_test_cases())) + def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: return ("uint_wrong_length", "uint", gen_suite.render_suite( title="UInt Wrong Length", From 13317d3dfc84320ca2bfe76c5a2eb2f783c524ba Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:31:42 +1000 Subject: [PATCH 56/61] Update test_generators/ssz/uint_test_cases.py Co-Authored-By: protolambda --- test_generators/ssz/uint_test_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz/uint_test_cases.py b/test_generators/ssz/uint_test_cases.py index 9ede168480..6d6492c9e9 100644 --- a/test_generators/ssz/uint_test_cases.py +++ b/test_generators/ssz/uint_test_cases.py @@ -30,7 +30,7 @@ def generate_random_uint_test_cases(): sedes = UInt(bit_size) for _ in range(RANDOM_TEST_CASES_PER_BIT_SIZE): - value = random.randrange(0, 2 ** bit_size) + value = random.randrange(0, 2**bit_size) serial = ssz.encode(value, sedes) # note that we need to create the tags in each loop cycle, otherwise ruamel will use # YAML references which makes the resulting file harder to read From 5e902b448a4c7533b5de21fcd913b6fdc6d0fb1d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:32:51 +1000 Subject: [PATCH 57/61] Update test_libs/config_helpers/README.md Co-Authored-By: protolambda --- test_libs/config_helpers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/config_helpers/README.md b/test_libs/config_helpers/README.md index 1844820825..eaa3f3b400 100644 --- a/test_libs/config_helpers/README.md +++ b/test_libs/config_helpers/README.md @@ -12,7 +12,7 @@ configs_path = 'configs/' import preset_loader from eth2spec.phase0 import spec -my_presets = preset_loader.load_presets(configs_path, 'main_net') +my_presets = preset_loader.load_presets(configs_path, 'mainnet') spec.apply_constants_preset(my_presets) ``` From dba7a1890b90f1351ed384180411b41433d18e81 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 15 Apr 2019 23:33:05 +1000 Subject: [PATCH 58/61] Update test_generators/ssz/main.py Co-Authored-By: protolambda --- test_generators/ssz/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_generators/ssz/main.py b/test_generators/ssz/main.py index e1c67cd9ed..1c09d51e75 100644 --- a/test_generators/ssz/main.py +++ b/test_generators/ssz/main.py @@ -30,6 +30,7 @@ def ssz_wrong_uint_suite(configs_path: str) -> gen_typing.TestSuiteOutput: handler="uint", test_cases=generate_uint_wrong_length_test_cases())) + def ssz_uint_bounds_suite(configs_path: str) -> gen_typing.TestSuiteOutput: return ("uint_bounds", "uint", gen_suite.render_suite( title="UInt Bounds", From c3d321ada80ce1bb6ae0f86205ca8ffd1b40f946 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Apr 2019 23:37:13 +1000 Subject: [PATCH 59/61] fix argument typing for auxilary transition func --- test_libs/pyspec/eth2spec/phase0/state_transition.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py index d25cd3aba0..9be192c1fb 100644 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase0/state_transition.py @@ -10,6 +10,7 @@ from .spec import ( BeaconState, BeaconBlock, + Slot ) @@ -98,7 +99,7 @@ def process_epoch_transition(state: BeaconState) -> None: spec.finish_epoch_update(state) -def state_transition_to(state: BeaconState, up_to: int) -> BeaconState: +def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: while state.slot < up_to: spec.cache_state(state) if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: From d2b7a8f5de8265884c1c112ddc8bcf9ac7e6c8ea Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Apr 2019 23:55:32 +1000 Subject: [PATCH 60/61] implement assertion sanity check suggested in PR --- test_generators/operations/deposits.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 3e47482a66..85c93f86b4 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -116,7 +116,13 @@ def invalid_deposit_index(): yield 'description', 'invalid deposit index' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) - yield 'post', None + try: + spec.process_deposit(state, new_dep) + except AssertionError: + # expected + yield 'post', None + return + raise Exception('invalid_deposit_index has unexpectedly allowed deposit') @to_dict @@ -128,7 +134,13 @@ def invalid_deposit_proof(): yield 'description', 'invalid deposit proof' yield 'pre', encode(state, spec.BeaconState) yield 'deposit', encode(new_dep, spec.Deposit) - yield 'post', None + try: + spec.process_deposit(state, new_dep) + except AssertionError: + # expected + yield 'post', None + return + raise Exception('invalid_deposit_index has unexpectedly allowed deposit') @to_tuple From 110af997cd779953a8d379070f268c067a74385a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Apr 2019 13:32:28 +1000 Subject: [PATCH 61/61] Update scripts/phase0/function_puller.py Co-Authored-By: protolambda --- scripts/phase0/function_puller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 812498b2bd..59e5b5e24a 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,7 +2,7 @@ from typing import List -def get_spec(file_name) -> List[str]: +def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None current_name = None