Mypy testing suite & various fixes

.mypy/mypy_check.py

@@ -0,0 +1,32 @@
+# This file is part of Scapy
+# See http://www.secdev.org/projects/scapy for more information
+# Copyright (C) Philippe Biondi <[email protected]>
+# Copyright (C) Gabriel Potter <[email protected]>
+# This program is published under a GPLv2 license
+
+"""
+Performs Static typing checks over Scapy's codebase
+"""
+
+import io
+import os
+import sys
+
+from mypy.main import main as mypy_main
+
+# Load files
+
+with io.open("./.mypy/mypy_enabled.txt") as fd:
+    FILES = [l.strip() for l in fd.readlines() if l.strip() and l[0] != "#"]
+
+if not FILES:
+    print("No files specified. Arborting")
+    sys.exit(0)
+
+# Generate Mypy arguments
+
+ARGS = ["--py2"] + [os.path.abspath(f) for f in FILES]
+
+# Run Mypy over the files
+
+mypy_main(None, sys.stdout, sys.stderr, ARGS)

.mypy/mypy_enabled.txt

@@ -0,0 +1,5 @@
+# This file registers all files that have already been processed as part of
+# https://github.com/secdev/scapy/issues/2158, and therefore will be enforced
+# with unit tests in future development.
+
+# Style cheet: https://mypy.readthedocs.io/en/latest/cheat_sheet.html

.travis.yml

@@ -6,7 +6,7 @@ matrix:
         - os: linux
           python: 3.6
           env:
-            - TOXENV=flake8,twine,docs,spell
+            - TOXENV=mypy,flake8,twine,docs,spell
 
         # Run as a regular user
         - os: linux

scapy/__init__.py

@@ -19,6 +19,7 @@
 
 
 def _version_from_git_describe():
+    # type: () -> str
     """
     Read the version from ``git describe``. It returns the latest tag with an
     optional suffix if the current directory is not exactly on the tag.
@@ -38,6 +39,9 @@ def _version_from_git_describe():
 
         >>> _version_from_git_describe()
         '2.3.2.dev346'
+
+    :raises CalledProcessError: if git is unavailable
+    :return: the scapy latest tag
     """
     if not os.path.isdir(os.path.join(os.path.dirname(_SCAPY_PKG_DIR), '.git')):  # noqa: E501
         raise ValueError('not in scapy git repo')
@@ -62,6 +66,11 @@ def _version_from_git_describe():
 
 
 def _version():
+    # () -> str
+    """Returns the Scapy version from multiple methods
+
+    :return: the scapy version
+    """
     version_file = os.path.join(_SCAPY_PKG_DIR, 'VERSION')
     try:
         tag = _version_from_git_describe()

scapy/main.py

@@ -52,6 +52,7 @@
 
 
 def _probe_config_file(cf):
+    # str -> str
     cf_path = os.path.join(os.path.expanduser("~"), cf)
     try:
         os.stat(cf_path)

scapy/tools/UTscapy.py

@@ -29,6 +29,13 @@
 from scapy.modules.six.moves import range
 from scapy.compat import base64_bytes, bytes_hex, plain_str
 
+ANNOTATIONS_MODE = False  # Set to True to use pyannotate
+
+if ANNOTATIONS_MODE:
+    try:
+        from pyannotate_runtime import collect_types
+    except ImportError:
+        raise ImportError("Please install pyannotate !")
 
 #   Util class   #
 
@@ -696,7 +703,7 @@ def usage():
 -T\t\t: if -t is used with *, remove a specific file (can be used many times)
 -l\t\t: generate local .js and .css files
 -F\t\t: expand only failed tests
--b\t\t: stop at first failed campaign
+-b\t\t: don't stop at the first failed campaign
 -d\t\t: dump campaign
 -D\t\t: dump campaign and stop
 -C\t\t: don't calculate CRC and SHA
@@ -780,6 +787,9 @@ def resolve_testfiles(TESTFILES):
 
 
 def main():
+    if ANNOTATIONS_MODE:
+        collect_types.init_types_collection()
+
     argv = sys.argv[1:]
     ignore_globals = list(six.moves.builtins.__dict__)
 
@@ -807,7 +817,7 @@ def main():
             if opt == "-h":
                 usage()
             elif opt == "-b":
-                BREAKFAILED =  True
+                BREAKFAILED = False
             elif opt == "-F":
                 ONLYFAILED = True
             elif opt == "-q":
@@ -885,6 +895,9 @@ def main():
         if six.PY2:
             KW_KO.append("python3_only")
 
+        if ANNOTATIONS_MODE:
+            collect_types.start()
+
         if VERB > 2:
             print("### Booting scapy...", file=sys.stderr)
         try:
@@ -964,6 +977,11 @@ def main():
 
     if VERB > 2:
         print("### Writing output...", file=sys.stderr)
+
+    if ANNOTATIONS_MODE:
+        collect_types.stop()
+        collect_types.dump_stats("pyannotate_results")
+
     # Concenate outputs
     if FORMAT == Format.HTML:
         glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL, glob_title)

test/regression.uts

@@ -737,7 +737,8 @@ r
 assert r in [
     b'u\x14\x00\x1c\xc2\xf6\x80\x00\xde\x01k\xd3\x7f\x00\x00\x01\x7f\x00\x00\x01y\xc9>\xa6\x84\xd8\xc2\xb7',
     b'E\xa7\x00\x1c\xb0c\xc0\x00\xf6\x01U\xd3\x7f\x00\x00\x01\x7f\x00\x00\x01\xfex\xb3\x92B<\x0b\xb8',
-    '\x85\x7f\x00\x1c\xc2\xf6\x00\x00\xde\x01\xdbh\x7f\x00\x00\x01\x7f\x00\x00\x01y\xc9>\xa6\x84\xd8\xc2\xb7'
+    b'\x85\x7f\x00\x1c\xc2\xf6\x00\x00\xde\x01\xdbh\x7f\x00\x00\x01\x7f\x00\x00\x01y\xc9>\xa6\x84\xd8\xc2\xb7',
+    b'\xf5H\x00\x1c\x0fr \x00\xff\x01\xde#\x7f\x00\x00\x01\x7f\x00\x00\x01\xfex\xb3\x92B<\x0b\xb8'
 ]
 
 = fuzz on packets with advanced RandNum
@@ -3653,7 +3654,8 @@ random.seed(0x2807)
 p = IPv6()/ICMPv6NIQueryIPv4(data="freebsd")
 h_py2 = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:y\xb2V<\x7f\x87\x14\xde'
 h_py3 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:\x88\xccb\x19~\x9e\xe3a'
-assert p.hashret() in [h_py2, h_py3]
+h_py3_2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:$#\xb5\xb7\xd0\xbf \xe2'
+assert p.hashret() in [h_py2, h_py3, h_py3_2]
 
 
 ############

test/run_tests_py2.bat

@@ -5,7 +5,7 @@ set PWD=%MYDIR%
 set PYTHONPATH=%MYDIR%
 set PYTHONDONTWRITEBYTECODE=True
 if [%1]==[] (
-  python "%MYDIR%\scapy\tools\UTscapy.py" -c configs\\windows2.utsc -o scapy_regression_test_%date:~6,4%_%date:~3,2%_%date:~0,2%.html
+  python "%MYDIR%\scapy\tools\UTscapy.py" -c configs\\windows2.utsc -b -o scapy_regression_test_%date:~6,4%_%date:~3,2%_%date:~0,2%.html
 ) else (
   python "%MYDIR%\scapy\tools\UTscapy.py" %*
 )

tox.ini

@@ -110,6 +110,14 @@ deps = codecov
 commands = codecov -e TOXENV
 
 
+[testenv:mypy]
+description = "Check Scapy compliance against static typing"
+skip_install = true
+deps = mypy
+       typing
+commands = python .mypy/mypy_check.py
+
+
 [testenv:docs]
 changedir = doc/scapy
 deps = sphinx