Merge pull request #1166 from ethereum/mypy

Add mypy type hinting check

.gitignore

@@ -9,7 +9,9 @@ build/
 output/
 
 eth2.0-spec-tests/
+
 .pytest_cache
+.mypy_cache
 
 # Dynamically built from Markdown spec
 test_libs/pyspec/eth2spec/phase0/spec.py

Makefile

@@ -59,7 +59,10 @@ open_cov:
 
 lint: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); . venv/bin/activate; \
-	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec;
+	flake8  --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; \
+	cd ./eth2spec; \
+	mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase0; \
+	mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs -p phase1
 
 install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS)
 	cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt

scripts/build_spec.py

@@ -13,9 +13,10 @@
 
 PHASE0_IMPORTS = '''from typing import (
     Any,
+    Callable,
     Dict,
     List,
-    NewType,
+    Set,
     Tuple,
 )
 
@@ -25,7 +26,8 @@
 )
 from eth2spec.utils.ssz.ssz_typing import (
     # unused: uint8, uint16, uint32, uint128, uint256,
-    uint64, Container, Vector, BytesN
+    uint64, Container, Vector,
+    Bytes4, Bytes32, Bytes48, Bytes96,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
@@ -38,9 +40,11 @@
 '''
 PHASE1_IMPORTS = '''from typing import (
     Any,
+    Callable,
     Dict,
     List,
-    NewType,
+    Optional,
+    Set,
     Tuple,
 )
 
@@ -52,7 +56,8 @@
 )
 from eth2spec.utils.ssz.ssz_typing import (
     # unused: uint8, uint16, uint32, uint128, uint256,
-    uint64, Container, Vector, BytesN
+    uint64, Container, Vector,
+    Bytes4, Bytes32, Bytes48, Bytes96,
 )
 from eth2spec.utils.bls import (
     bls_aggregate_pubkeys,
@@ -62,13 +67,6 @@
 
 from eth2spec.utils.hash_function import hash
 '''
-NEW_TYPES = {
-    'Slot': 'int',
-    'Epoch': 'int',
-    'Shard': 'int',
-    'ValidatorIndex': 'int',
-    'Gwei': 'int',
-}
 BYTE_TYPES = [4, 32, 48, 96]
 SUNDRY_FUNCTIONS = '''
 def get_ssz_type_by_name(name: str) -> Container:
@@ -77,36 +75,33 @@ def get_ssz_type_by_name(name: str) -> Container:
 
 # Monkey patch validator compute committee code
 _compute_committee = compute_committee
-committee_cache = {}
+committee_cache: Dict[Tuple[Hash, Hash, int, int], List[ValidatorIndex]] = {}
 
 
-def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]:
+def compute_committee(indices: List[ValidatorIndex],  # type: ignore
+                      seed: Hash,
+                      index: int,
+                      count: int) -> List[ValidatorIndex]:
     param_hash = (hash_tree_root(indices), seed, index, count)
 
-    if param_hash in committee_cache:
-        return committee_cache[param_hash]
-    else:
-        ret = _compute_committee(indices, seed, index, count)
-        committee_cache[param_hash] = ret
-        return ret
+    if param_hash not in committee_cache:
+        committee_cache[param_hash] = _compute_committee(indices, seed, index, count)
+    return committee_cache[param_hash]
 
 
 # Monkey patch hash cache
 _hash = hash
-hash_cache = {}
+hash_cache: Dict[bytes, Hash] = {}
 
 
-def hash(x):
-    if x in hash_cache:
-        return hash_cache[x]
-    else:
-        ret = _hash(x)
-        hash_cache[x] = ret
-        return ret
+def hash(x: bytes) -> Hash:
+    if x not in hash_cache:
+        hash_cache[x] = Hash(_hash(x))
+    return hash_cache[x]
 
 
 # Access to overwrite spec constants based on configuration
-def apply_constants_preset(preset: Dict[str, Any]):
+def apply_constants_preset(preset: Dict[str, Any]) -> None:
     global_vars = globals()
     for k, v in preset.items():
         global_vars[k] = v
@@ -120,32 +115,39 @@ def apply_constants_preset(preset: Dict[str, Any]):
 
 
 def objects_to_spec(functions: Dict[str, str],
+                    custom_types: Dict[str, str],
                     constants: Dict[str, str],
                     ssz_objects: Dict[str, str],
                     inserts: Dict[str, str],
                     imports: Dict[str, str],
-                    new_types: Dict[str, str],
-                    byte_types: List[int],
                     ) -> str:
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
     """
-    new_type_definitions = '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in byte_types])
-    new_type_definitions += '\n' + '\n'.join(['Hash = Bytes32', 'BLSPubkey = Bytes48', 'BLSSignature = Bytes96'])
-    new_type_definitions += \
-        '\n' + '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()])
+    new_type_definitions = (
+        '\n\n'.join(
+            [
+                f"class {key}({value}):\n"
+                f"    def __init__(self, _x: {value}) -> None:\n"
+                f"        ...\n"
+                if value.startswith("uint")
+                else f"class {key}({value}):\n    pass\n"
+                for key, value in custom_types.items()
+            ]
+        )
+    )
     functions_spec = '\n\n'.join(functions.values())
     constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants))
     ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values())
     ssz_objects_reinitialization_spec = (
-        'def init_SSZ_types():\n    global_vars = globals()\n\n    '
+        'def init_SSZ_types() -> None:\n    global_vars = globals()\n\n    '
         + '\n\n    '.join([re.sub(r'(?!\n\n)\n', r'\n    ', value[:-1]) for value in ssz_objects.values()])
         + '\n\n'
         + '\n'.join(map(lambda x: '    global_vars[\'%s\'] = %s' % (x, x), ssz_objects.keys()))
     )
     spec = (
         imports
-        + '\n' + new_type_definitions
+        + '\n\n' + new_type_definitions
         + '\n\n' + constants_spec
         + '\n\n\n' + ssz_objects_instantiation_spec
         + '\n\n' + functions_spec
@@ -171,7 +173,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st
     return old_constants
 
 
-def dependency_order_ssz_objects(objects: Dict[str, str]) -> None:
+def dependency_order_ssz_objects(objects: Dict[str, str], custom_types: Dict[str, str]) -> None:
     """
     Determines which SSZ Object is depenedent on which other and orders them appropriately
     """
@@ -180,14 +182,14 @@ def dependency_order_ssz_objects(objects: Dict[str, str]) -> None:
         dependencies = re.findall(r'(: [A-Z][\w[]*)', value)
         dependencies = map(lambda x: re.sub(r'\W|Vector|List|Container|Hash|BLSPubkey|BLSSignature|uint\d+|Bytes\d+|bytes', '', x), dependencies)
         for dep in dependencies:
-            if dep in NEW_TYPES or len(dep) == 0:
+            if dep in custom_types or len(dep) == 0:
                 continue
             key_list = list(objects.keys())
             for item in [dep, key] + key_list[key_list.index(dep)+1:]:
                 objects[item] = objects.pop(item)
 
 
-def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]) -> Dict[str, str]:
+def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str], custom_types) -> Dict[str, str]:
     """
     Takes in old spec and new spec ssz objects, combines them,
     and returns the newer versions of the objects in dependency order.
@@ -199,7 +201,7 @@ def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]
             # remove leading variable name
             value = re.sub(r'^class [\w]*\(Container\):\n', '', value)
         old_objects[key] = old_objects.get(key, '') + value
-    dependency_order_ssz_objects(old_objects)
+    dependency_order_ssz_objects(old_objects, custom_types)
     return old_objects
 
 
@@ -211,18 +213,19 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     """
     Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function.
     """
-    functions0, constants0, ssz_objects0, inserts0 = spec0
-    functions1, constants1, ssz_objects1, inserts1 = spec1
+    functions0, custom_types0, constants0, ssz_objects0, inserts0 = spec0
+    functions1, custom_types1, constants1, ssz_objects1, inserts1 = spec1
     functions = combine_functions(functions0, functions1)
+    custom_types = combine_constants(custom_types0, custom_types1)
     constants = combine_constants(constants0, constants1)
-    ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1)
+    ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1, custom_types)
     inserts = combine_inserts(inserts0, inserts1)
-    return functions, constants, ssz_objects, inserts
+    return functions, custom_types, constants, ssz_objects, inserts
 
 
 def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]:
-    functions, constants, ssz_objects, inserts = get_spec(sourcefile)
-    spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES)
+    functions, custom_types, constants, ssz_objects, inserts = get_spec(sourcefile)
+    spec = objects_to_spec(functions, custom_types, constants, ssz_objects, inserts, PHASE0_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)
@@ -239,7 +242,7 @@ def build_phase1_spec(phase0_sourcefile: str,
     spec_objects = phase0_spec
     for value in [phase1_custody, phase1_shard_data]:
         spec_objects = combine_spec_objects(spec_objects, value)
-    spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, NEW_TYPES, BYTE_TYPES)
+    spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS)
     if outfile is not None:
         with open(outfile, 'w') as out:
             out.write(spec)

scripts/function_puller.py

@@ -29,6 +29,7 @@ def get_spec(file_name: str) -> SpecObject:
     inserts = {}
     function_matcher = re.compile(FUNCTION_REGEX)
     inserts_matcher = re.compile(BEGIN_INSERT_REGEX)
+    custom_types = {}
     for linenum, line in enumerate(open(file_name).readlines()):
         line = line.rstrip()
         if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`':
@@ -64,20 +65,23 @@ def get_spec(file_name: str) -> SpecObject:
                     ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n'
                 else:
                     functions[current_name] = functions.get(current_name, '') + line + '\n'
-            # Handle constant table entries
+            # Handle constant and custom types table entries
             elif pulling_from is None and len(line) > 0 and line[0] == '|':
                 row = line[1:].split('|')
                 if len(row) >= 2:
                     for i in range(2):
                         row[i] = row[i].strip().strip('`')
                         if '`' in row[i]:
                             row[i] = row[i][:row[i].find('`')]
-                    eligible = True
-                    if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_':
-                        eligible = False
-                    for c in row[0]:
-                        if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789':
+                    if row[1].startswith('uint') or row[1].startswith('Bytes'):
+                        custom_types[row[0]] = row[1]
+                    else:
+                        eligible = True
+                        if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_':
                             eligible = False
-                    if eligible:
-                        constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890')
-    return functions, constants, ssz_objects, inserts
+                        for c in row[0]:
+                            if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789':
+                                eligible = False
+                        if eligible:
+                            constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890')
+    return functions, custom_types, constants, ssz_objects, inserts