Merge branch 'dev' into test_genesis

Makefile

@@ -59,10 +59,9 @@ open_cov:
 
 lint: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); . venv/bin/activate; \
-	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \
-	cd ./eth2spec; \
-	mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0; \
-	mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1
+	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec \
+	&& cd ./eth2spec && mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0 \
+	&& mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1;
 
 install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS)
 	cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt

configs/constant_presets/mainnet.yaml

@@ -79,6 +79,10 @@ MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4
 EPOCHS_PER_HISTORICAL_VECTOR: 65536
 # 2**13 (= 8,192) epochs ~36 days
 EPOCHS_PER_SLASHED_BALANCES_VECTOR: 8192
+# 2**24 (= 16,777,216) historical roots, ~26,131 years
+HISTORICAL_ROOTS_LIMIT: 16777216
+# 2**40 (= 1,099,511,627,776) validator spots
+VALIDATOR_REGISTRY_LIMIT: 1099511627776
 
 
 # Reward and penalty quotients

configs/constant_presets/minimal.yaml

@@ -80,6 +80,10 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096
 EPOCHS_PER_HISTORICAL_VECTOR: 64
 # [customized] smaller state
 EPOCHS_PER_SLASHED_BALANCES_VECTOR: 64
+# 2**24 (= 16,777,216) historical roots
+HISTORICAL_ROOTS_LIMIT: 16777216
+# 2**40 (= 1,099,511,627,776) validator spots
+VALIDATOR_REGISTRY_LIMIT: 1099511627776
 
 
 # Reward and penalty quotients

scripts/README.md

@@ -1,18 +1,18 @@
 # Building pyspecs from specs.md
 
-The benefit of the particular spec design is that the given markdown files can be converted to a `spec.py` file for the purposes of testing and linting. The result of this is that bugs are discovered and patched more quickly.
+The benefit of the particular spec design is that the given Markdown files can be converted to a `spec.py` file for the purposes of testing and linting. As a result, bugs are discovered and patched more quickly.
 
-Specs can be built from either a single markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`.
+Specs can be built from either a single Markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`.
 
 ## Usage
 
-For usage of the spec builder run `python3 -m build_spec --help`.
+For usage of the spec builder, run `python3 -m build_spec --help`.
 
 ## `@Labels` and inserts
 
 The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows:
 
-* In the old file, a label is added as a python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`:
+* In the old file, a label is added as a Python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`:
 
 ```python
 def foo(x):
@@ -21,12 +21,12 @@ def foo(x):
     return x
 ```
 
-* In spec1, the new code could then be inserted by having a code-block that looked as follows:
+* In spec1, the new code can then be inserted by having a code-block that looks as follows:
 
 ```python
 #begin insert @YourLabelHere
     x += x
 #end insert @YourLabelHere
 ```
 
-**Note** that the code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point.
+*Note*: The code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point.

scripts/build_spec.py

@@ -12,13 +12,7 @@
 
 
 PHASE0_IMPORTS = '''from typing import (
-    Any,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Set,
-    Tuple,
+    Any, Callable, Dict, List, Optional, Set, Sequence, Tuple,
 )
 
 from dataclasses import (
@@ -31,27 +25,19 @@
     signing_root,
 )
 from eth2spec.utils.ssz.ssz_typing import (
-    # unused: uint8, uint16, uint32, uint128, uint256,
-    uint64, Container, Vector,
+    Bit, Bool, Container, List, Vector, Bytes, uint64,
     Bytes4, Bytes32, Bytes48, Bytes96,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
     bls_verify,
     bls_verify_multiple,
 )
-# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation.
 
 from eth2spec.utils.hash_function import hash
 '''
 PHASE1_IMPORTS = '''from typing import (
-    Any,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Set,
-    Tuple,
+    Any, Callable, Dict, Optional, Set, Sequence, MutableSequence, Tuple,
 )
 
 from dataclasses import (
@@ -66,8 +52,7 @@
     is_empty,
 )
 from eth2spec.utils.ssz.ssz_typing import (
-    # unused: uint8, uint16, uint32, uint128, uint256,
-    uint64, Container, Vector,
+    Bit, Bool, Container, List, Vector, Bytes, uint64,
     Bytes4, Bytes32, Bytes48, Bytes96,
 )
 from eth2spec.utils.bls import (
@@ -78,39 +63,34 @@
 
 from eth2spec.utils.hash_function import hash
 '''
-BYTE_TYPES = [4, 32, 48, 96]
 SUNDRY_FUNCTIONS = '''
-def get_ssz_type_by_name(name: str) -> Container:
-    return globals()[name]
+# Monkey patch hash cache
+_hash = hash
+hash_cache: Dict[bytes, Hash] = {}
+
+
+def hash(x: bytes) -> Hash:
+    if x not in hash_cache:
+        hash_cache[x] = Hash(_hash(x))
+    return hash_cache[x]
 
 
 # Monkey patch validator compute committee code
 _compute_committee = compute_committee
-committee_cache: Dict[Tuple[Hash, Hash, int, int], List[ValidatorIndex]] = {}
+committee_cache: Dict[Tuple[Hash, Hash, int, int], Sequence[ValidatorIndex]] = {}
 
 
-def compute_committee(indices: List[ValidatorIndex],  # type: ignore
+def compute_committee(indices: Sequence[ValidatorIndex],  # type: ignore
                       seed: Hash,
                       index: int,
-                      count: int) -> List[ValidatorIndex]:
-    param_hash = (hash_tree_root(indices), seed, index, count)
+                      count: int) -> Sequence[ValidatorIndex]:
+    param_hash = (hash(b''.join(index.to_bytes(length=4, byteorder='little') for index in indices)), seed, index, count)
 
     if param_hash not in committee_cache:
         committee_cache[param_hash] = _compute_committee(indices, seed, index, count)
     return committee_cache[param_hash]
 
 
-# Monkey patch hash cache
-_hash = hash
-hash_cache: Dict[bytes, Hash] = {}
-
-
-def hash(x: bytes) -> Hash:
-    if x not in hash_cache:
-        hash_cache[x] = Hash(_hash(x))
-    return hash_cache[x]
-
-
 # Access to overwrite spec constants based on configuration
 def apply_constants_preset(preset: Dict[str, Any]) -> None:
     global_vars = globals()
@@ -125,6 +105,18 @@ def apply_constants_preset(preset: Dict[str, Any]) -> None:
 '''
 
 
+def strip_comments(raw: str) -> str:
+    comment_line_regex = re.compile('^\s+# ')
+    lines = raw.split('\n')
+    out = []
+    for line in lines:
+        if not comment_line_regex.match(line):
+            if '  #' in line:
+                line = line[:line.index('  #')]
+            out.append(line)
+    return '\n'.join(out)
+
+
 def objects_to_spec(functions: Dict[str, str],
                     custom_types: Dict[str, str],
                     constants: Dict[str, str],
@@ -138,11 +130,7 @@ def objects_to_spec(functions: Dict[str, str],
     new_type_definitions = (
         '\n\n'.join(
             [
-                f"class {key}({value}):\n"
-                f"    def __init__(self, _x: {value}) -> None:\n"
-                f"        ...\n"
-                if value.startswith("uint")
-                else f"class {key}({value}):\n    pass\n"
+                f"class {key}({value}):\n    pass\n"
                 for key, value in custom_types.items()
             ]
         )
@@ -152,7 +140,8 @@ def objects_to_spec(functions: Dict[str, str],
     ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values())
     ssz_objects_reinitialization_spec = (
         'def init_SSZ_types() -> None:\n    global_vars = globals()\n\n    '
-        + '\n\n    '.join([re.sub(r'(?!\n\n)\n', r'\n    ', value[:-1]) for value in ssz_objects.values()])
+        + '\n\n    '.join([strip_comments(re.sub(r'(?!\n\n)\n', r'\n    ', value[:-1]))
+                           for value in ssz_objects.values()])
         + '\n\n'
         + '\n'.join(map(lambda x: '    global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys()))
     )
@@ -184,17 +173,32 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st
     return old_constants
 
 
+ignored_dependencies = [
+    'Bit', 'Bool', 'Vector', 'List', 'Container', 'Hash', 'BLSPubkey', 'BLSSignature', 'Bytes', 'BytesN'
+    'Bytes4', 'Bytes32', 'Bytes48', 'Bytes96',
+    'uint8', 'uint16', 'uint32', 'uint64', 'uint128', 'uint256',
+    'bytes'  # to be removed after updating spec doc
+]
+
+
 def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None:
     """
     Determines which SSZ Object is depenedent on which other and orders them appropriately
     """
     items = list(objects.items())
     for key, value in items:
-        dependencies = re.findall(r'(: [A-Z][\w[]*)', value)
-        dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|Bytes\d+|bytes', '', x), dependencies)
+        dependencies = []
+        for line in value.split('\n'):
+            if not re.match(r'\s+\w+: .+', line):
+                continue  # skip whitespace etc.
+            line = line[line.index(':') + 1:]  # strip of field name
+            if '#' in line:
+                line = line[:line.index('#')]  # strip of comment
+            dependencies.extend(re.findall(r'(\w+)', line))  # catch all legible words, potential dependencies
+        dependencies = filter(lambda x: '_' not in x and x.upper() != x, dependencies)  # filter out constants
+        dependencies = filter(lambda x: x not in ignored_dependencies, dependencies)
+        dependencies = filter(lambda x: x not in custom_types, dependencies)
         for dep in dependencies:
-            if dep in custom_types or len(dep) == 0:
-                continue
             key_list = list(objects.keys())
             for item in [dep, key] + key_list[key_list.index(dep)+1:]:
                 objects[item] = objects.pop(item)