From 69ad3fcbe58256f2fafa9f605ba2188383f0b865 Mon Sep 17 00:00:00 2001 From: gpotter2 Date: Mon, 22 Jul 2019 17:24:48 +0200 Subject: [PATCH 1/2] Mypy testing suite & various fixes --- .appveyor.yml | 4 +- .../appveyor}/InstallNpcap.ps1 | 0 .../appveyor}/InstallWindumpNpcap.ps1 | 0 {.travis => .config}/codespell_ignore.txt | 0 .config/mypy/mypy.ini | 10 +++ .config/mypy/mypy_check.py | 50 ++++++++++++ .config/mypy/mypy_enabled.txt | 9 +++ {.travis => .config/travis}/install.sh | 8 +- {.travis => .config/travis}/test.sh | 0 .travis.yml | 6 +- scapy/__init__.py | 13 ++- scapy/compat.py | 29 +++++++ scapy/contrib/http2.py | 77 +++++++++--------- scapy/main.py | 80 +++++++++++++------ scapy/tools/UTscapy.py | 24 +++++- scapy/tools/scapy_pyannotate.py | 21 +++++ test/regression.uts | 6 +- test/run_tests_py2.bat | 2 +- tox.ini | 11 ++- 19 files changed, 265 insertions(+), 85 deletions(-) rename {.appveyor => .config/appveyor}/InstallNpcap.ps1 (100%) rename {.appveyor => .config/appveyor}/InstallWindumpNpcap.ps1 (100%) rename {.travis => .config}/codespell_ignore.txt (100%) create mode 100644 .config/mypy/mypy.ini create mode 100644 .config/mypy/mypy_check.py create mode 100644 .config/mypy/mypy_enabled.txt rename {.travis => .config/travis}/install.sh (81%) rename {.travis => .config/travis}/test.sh (100%) create mode 100644 scapy/tools/scapy_pyannotate.py diff --git a/.appveyor.yml b/.appveyor.yml index 207e4e42961..ddbfde1f7a5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -31,8 +31,8 @@ build: off install: # Install the npcap, windump and wireshark suites - - ps: .\.appveyor\InstallNpcap.ps1 - - ps: .\.appveyor\InstallWindumpNpcap.ps1 + - ps: .\.config\appveyor\InstallNpcap.ps1 + - ps: .\.config\appveyor\InstallWindumpNpcap.ps1 # Installs Wireshark 3.0 (and its dependencies) # https://github.com/mkevenaar/chocolatey-packages/issues/16 - choco install -n KB3033929 KB2919355 kb2999226 diff --git a/.appveyor/InstallNpcap.ps1 b/.config/appveyor/InstallNpcap.ps1 similarity index 100% rename from .appveyor/InstallNpcap.ps1 rename to .config/appveyor/InstallNpcap.ps1 diff --git a/.appveyor/InstallWindumpNpcap.ps1 b/.config/appveyor/InstallWindumpNpcap.ps1 similarity index 100% rename from .appveyor/InstallWindumpNpcap.ps1 rename to .config/appveyor/InstallWindumpNpcap.ps1 diff --git a/.travis/codespell_ignore.txt b/.config/codespell_ignore.txt similarity index 100% rename from .travis/codespell_ignore.txt rename to .config/codespell_ignore.txt diff --git a/.config/mypy/mypy.ini b/.config/mypy/mypy.ini new file mode 100644 index 00000000000..32bfce7fa5e --- /dev/null +++ b/.config/mypy/mypy.ini @@ -0,0 +1,10 @@ +[mypy] + +[mypy-IPython] +ignore_missing_imports = True + +[mypy-colorama] +ignore_missing_imports = True + +[mypy-traitlets.config.loader] +ignore_missing_imports = True \ No newline at end of file diff --git a/.config/mypy/mypy_check.py b/.config/mypy/mypy_check.py new file mode 100644 index 00000000000..cb313e6e69d --- /dev/null +++ b/.config/mypy/mypy_check.py @@ -0,0 +1,50 @@ +# This file is part of Scapy +# See http://www.secdev.org/projects/scapy for more information +# Copyright (C) Philippe Biondi +# Copyright (C) Gabriel Potter +# This program is published under a GPLv2 license + +""" +Performs Static typing checks over Scapy's codebase +""" + +# IMPORTANT NOTE +# +# Because we are rolling out mypy tests progressively, +# we currently use --follow-imports=skip. This means that +# mypy doesn't check consistency between the imports (different files). +# +# Once each file has been processed individually, we'll remove that to +# check the inconsistencies across the files + +import io +import os +import sys + +from mypy.main import main as mypy_main + +# Load files + +with io.open("./.config/mypy/mypy_enabled.txt") as fd: + FILES = [l.strip() for l in fd.readlines() if l.strip() and l[0] != "#"] + +if not FILES: + print("No files specified. Arborting") + sys.exit(0) + +# Generate mypy arguments + +ARGS = [ + "--py2", + "--follow-imports=skip", + "--config-file=" + os.path.abspath( + os.path.join( + os.path.split(__file__)[0], + "mypy.ini" + ) + ) +] + [os.path.abspath(f) for f in FILES] + +# Run mypy over the files + +mypy_main(None, sys.stdout, sys.stderr, ARGS) diff --git a/.config/mypy/mypy_enabled.txt b/.config/mypy/mypy_enabled.txt new file mode 100644 index 00000000000..82462ed857d --- /dev/null +++ b/.config/mypy/mypy_enabled.txt @@ -0,0 +1,9 @@ +# This file registers all files that have already been processed as part of +# https://github.com/secdev/scapy/issues/2158, and therefore will be enforced +# with unit tests in future development. + +# Style cheet: https://mypy.readthedocs.io/en/latest/cheat_sheet.html + +scapy/__init__.py +scapy/main.py +scapy/contrib/http2.py \ No newline at end of file diff --git a/.travis/install.sh b/.config/travis/install.sh similarity index 81% rename from .travis/install.sh rename to .config/travis/install.sh index 9938f41a173..607e0c572a1 100644 --- a/.travis/install.sh +++ b/.config/travis/install.sh @@ -2,13 +2,11 @@ # Install on osx if [ "$TRAVIS_OS_NAME" = "osx" ] then - pip3 install tox if [ ! -z $SCAPY_USE_PCAPDNET ] then brew update brew install libdnet libpcap fi - exit 0 fi # Install wireshark data @@ -25,11 +23,11 @@ then $SCAPY_SUDO apt-get -qy install libdumbnet-dev libpcap-dev fi -# Check pip -sudo pip install --upgrade pip setuptools --ignore-installed +# Update pip & setuptools (tox uses those) +python -m pip install --upgrade pip setuptools --ignore-installed # Make sure tox is installed and up to date -pip install -U tox --ignore-installed +python -m pip install -U tox --ignore-installed # Dump Environment (so that we can check PATH, UT_FLAGS, etc.) set diff --git a/.travis/test.sh b/.config/travis/test.sh similarity index 100% rename from .travis/test.sh rename to .config/travis/test.sh diff --git a/.travis.yml b/.travis.yml index ad8fb59da22..088825e17f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: - os: linux python: 3.6 env: - - TOXENV=flake8,twine,docs,spell + - TOXENV=mypy,flake8,twine,docs,spell # Run as a regular user - os: linux @@ -130,7 +130,7 @@ matrix: - TOXENV=linux_warnings install: - - bash .travis/install.sh + - bash .config/travis/install.sh - python -c "from scapy.all import conf; print(repr(conf))" -script: bash .travis/test.sh +script: bash .config/travis/test.sh diff --git a/scapy/__init__.py b/scapy/__init__.py index d5ad6164ba4..802edbd4596 100644 --- a/scapy/__init__.py +++ b/scapy/__init__.py @@ -19,6 +19,7 @@ def _version_from_git_describe(): + # type: () -> str """ Read the version from ``git describe``. It returns the latest tag with an optional suffix if the current directory is not exactly on the tag. @@ -38,6 +39,9 @@ def _version_from_git_describe(): >>> _version_from_git_describe() '2.3.2.dev346' + + :raises CalledProcessError: if git is unavailable + :return: Scapy's latest tag """ if not os.path.isdir(os.path.join(os.path.dirname(_SCAPY_PKG_DIR), '.git')): # noqa: E501 raise ValueError('not in scapy git repo') @@ -62,6 +66,11 @@ def _version_from_git_describe(): def _version(): + # type: () -> str + """Returns the Scapy version from multiple methods + + :return: the Scapy version + """ version_file = os.path.join(_SCAPY_PKG_DIR, 'VERSION') try: tag = _version_from_git_describe() @@ -91,7 +100,9 @@ def _version(): VERSION = __version__ = _version() -VERSION_MAIN = re.search(r"[0-9.]+", VERSION).group() + +_tmp = re.search(r"[0-9.]+", VERSION) +VERSION_MAIN = _tmp.group() if _tmp is not None else VERSION if __name__ == "__main__": from scapy.main import interact diff --git a/scapy/compat.py b/scapy/compat.py index 8d06a67033d..5bbbb88c271 100644 --- a/scapy/compat.py +++ b/scapy/compat.py @@ -117,3 +117,32 @@ def gzip_compress(x): else: gzip_decompress = gzip.decompress gzip_compress = gzip.compress + +# Typing compatibility + +try: + # Only required if using mypy-lang for static typing + from typing import Optional, List, Union, Callable, Any, Tuple, Sized, \ + Dict, Pattern, cast +except ImportError: + # Let's make some fake ones. + + def cast(_type, obj): + return obj + + class _FakeType(object): + # make the objects subscriptable indefinetly + def __getitem__(self, item): + return _FakeType() + + Optional = _FakeType() + Union = _FakeType() + Callable = _FakeType() + List = _FakeType() + Dict = _FakeType() + Any = _FakeType() + Tuple = _FakeType() + Pattern = _FakeType() + + class Sized(object): + pass diff --git a/scapy/contrib/http2.py b/scapy/contrib/http2.py index a867ec89d3f..e41380baa09 100644 --- a/scapy/contrib/http2.py +++ b/scapy/contrib/http2.py @@ -41,11 +41,9 @@ # Only required if using mypy-lang for static typing # Most symbols are used in mypy-interpreted "comments". # Sized must be one of the superclasses of a class implementing __len__ -try: - from typing import Optional, List, Union, Callable, Any, Tuple, Sized # noqa: F401, E501 -except ImportError: - class Sized(object): - pass +from scapy.compat import Optional, List, Union, Callable, Any, \ + Tuple, Sized, Pattern # noqa: F401 +from scapy.base_classes import Packet_metaclass # noqa: F401 import scapy.fields as fields import scapy.packet as packet @@ -654,7 +652,7 @@ def _compute_value(self, pkt): ABC = abc.ABCMeta('ABC', (), {}) -class HPackStringsInterface(ABC, Sized): +class HPackStringsInterface(ABC, Sized): # type: ignore @abc.abstractmethod def __str__(self): pass @@ -1012,7 +1010,7 @@ class HPackZString(HPackStringsInterface): (0x3fffffff, 30) ] - static_huffman_tree = None + static_huffman_tree = None # type: HuffmanNode @classmethod def _huffman_encode_char(cls, c): @@ -1330,7 +1328,7 @@ class HPackHdrString(packet.Packet): ] def guess_payload_class(self, payload): - # type: (str) -> base_classes.Packet_metaclass + # type: (str) -> Packet_metaclass # Trick to tell scapy that the remaining bytes of the currently # dissected string is not a payload of this packet but of some other # underlayer packet @@ -1353,7 +1351,7 @@ class HPackHeaders(packet.Packet): """ @classmethod def dispatch_hook(cls, s=None, *_args, **_kwds): - # type: (Optional[str], *Any, **Any) -> base_classes.Packet_metaclass + # type: (Optional[str], *Any, **Any) -> Packet_metaclass """dispatch_hook returns the subclass of HPackHeaders that must be used to dissect the string. """ @@ -1369,7 +1367,7 @@ def dispatch_hook(cls, s=None, *_args, **_kwds): return HPackLitHdrFldWithoutIndexing def guess_payload_class(self, payload): - # type: (str) -> base_classes.Packet_metaclass + # type: (str) -> Packet_metaclass return config.conf.padding_layer @@ -1759,7 +1757,7 @@ class H2Setting(packet.Packet): ] def guess_payload_class(self, payload): - # type: (str) -> base_classes.Packet_metaclass + # type: (str) -> Packet_metaclass return config.conf.padding_layer @@ -2006,7 +2004,7 @@ class H2Frame(packet.Packet): ] def guess_payload_class(self, payload): - # type: (str) -> base_classes.Packet_metaclass + # type: (str) -> Packet_metaclass """ guess_payload_class returns the Class object to use for parsing a payload This function uses the H2Frame.type field value to decide which payload to parse. The implement cannot be # noqa: E501 performed using the simple bind_layers helper because sometimes the selection of which Class object to return # noqa: E501 @@ -2100,7 +2098,7 @@ class H2Seq(packet.Packet): ] def guess_payload_class(self, payload): - # type: (str) -> base_classes.Packet_metaclass + # type: (str) -> Packet_metaclass return config.conf.padding_layer @@ -2272,7 +2270,7 @@ class HPackHdrTable(Sized): # The value of this variable cannot be determined at declaration time. It is # noqa: E501 # initialized by an init_static_table call - _static_entries_last_idx = None + _static_entries_last_idx = None # type: int @classmethod def init_static_table(cls): @@ -2286,7 +2284,7 @@ def __init__(self, dynamic_table_max_size=4096, dynamic_table_cap_size=4096): # :param int dynamic_table_cap_size: the maximum-maximum size of the dynamic entry table in bytes # noqa: E501 :raises:s AssertionError """ - self._regexp = None + self._regexp = None # type: Pattern if isinstance(type(self)._static_entries_last_idx, type(None)): type(self).init_static_table() @@ -2514,7 +2512,6 @@ def gen_txt_repr(self, hdrs, register=True): @staticmethod def _optimize_header_length_and_packetify(s): - # type: (str) -> HPackHdrString # type: (str) -> HPackHdrString zs = HPackZString(s) if len(zs) >= len(s): @@ -2538,81 +2535,81 @@ def _convert_a_header_to_a_h2_header(self, hdr_name, hdr_value, is_sensitive, sh # The value is not indexed for this headers - hdr_value = self._optimize_header_length_and_packetify(hdr_value) + _hdr_value = self._optimize_header_length_and_packetify(hdr_value) # Searching if the header name is indexed idx = self.get_idx_by_name(hdr_name) if idx is not None: if is_sensitive( hdr_name, - hdr_value.getfieldval('data').origin() + _hdr_value.getfieldval('data').origin() ): return HPackLitHdrFldWithoutIndexing( never_index=1, index=idx, - hdr_value=hdr_value + hdr_value=_hdr_value ), len( HPackHdrEntry( self[idx].name(), - hdr_value.getfieldval('data').origin() + _hdr_value.getfieldval('data').origin() ) ) if should_index(hdr_name): return HPackLitHdrFldWithIncrIndexing( index=idx, - hdr_value=hdr_value + hdr_value=_hdr_value ), len( HPackHdrEntry( self[idx].name(), - hdr_value.getfieldval('data').origin() + _hdr_value.getfieldval('data').origin() ) ) return HPackLitHdrFldWithoutIndexing( index=idx, - hdr_value=hdr_value + hdr_value=_hdr_value ), len( HPackHdrEntry( self[idx].name(), - hdr_value.getfieldval('data').origin() + _hdr_value.getfieldval('data').origin() ) ) - hdr_name = self._optimize_header_length_and_packetify(hdr_name) + _hdr_name = self._optimize_header_length_and_packetify(hdr_name) if is_sensitive( - hdr_name.getfieldval('data').origin(), - hdr_value.getfieldval('data').origin() + _hdr_name.getfieldval('data').origin(), + _hdr_value.getfieldval('data').origin() ): return HPackLitHdrFldWithoutIndexing( never_index=1, index=0, - hdr_name=hdr_name, - hdr_value=hdr_value + hdr_name=_hdr_name, + hdr_value=_hdr_value ), len( HPackHdrEntry( - hdr_name.getfieldval('data').origin(), - hdr_value.getfieldval('data').origin() + _hdr_name.getfieldval('data').origin(), + _hdr_value.getfieldval('data').origin() ) ) - if should_index(hdr_name.getfieldval('data').origin()): + if should_index(_hdr_name.getfieldval('data').origin()): return HPackLitHdrFldWithIncrIndexing( index=0, - hdr_name=hdr_name, - hdr_value=hdr_value + hdr_name=_hdr_name, + hdr_value=_hdr_value ), len( HPackHdrEntry( - hdr_name.getfieldval('data').origin(), - hdr_value.getfieldval('data').origin() + _hdr_name.getfieldval('data').origin(), + _hdr_value.getfieldval('data').origin() ) ) return HPackLitHdrFldWithoutIndexing( index=0, - hdr_name=hdr_name, - hdr_value=hdr_value + hdr_name=_hdr_name, + hdr_value=_hdr_value ), len( HPackHdrEntry( - hdr_name.getfieldval('data').origin(), - hdr_value.getfieldval('data').origin() + _hdr_name.getfieldval('data').origin(), + _hdr_value.getfieldval('data').origin() ) ) diff --git a/scapy/main.py b/scapy/main.py index ea78d40ab45..13224232db2 100644 --- a/scapy/main.py +++ b/scapy/main.py @@ -31,9 +31,11 @@ from scapy.themes import DefaultTheme, BlackAndWhite, apply_ipython_style from scapy.consts import WINDOWS +from scapy.compat import cast, Any, Dict, List, Optional, Union + IGNORED = list(six.moves.builtins.__dict__) -GLOBKEYS = [] +GLOBKEYS = [] # type: List[str] LAYER_ALIASES = { "tls": "tls.all" @@ -52,6 +54,7 @@ def _probe_config_file(cf): + # type: (str) -> Union[str, None] cf_path = os.path.join(os.path.expanduser("~"), cf) try: os.stat(cf_path) @@ -62,6 +65,7 @@ def _probe_config_file(cf): def _read_config_file(cf, _globals=globals(), _locals=locals(), interactive=True): # noqa: E501 + # type: (str, Dict[str, Any], Dict[str, Any], bool) -> None """Read a config file: execute a python file while loading scapy, that may contain # noqa: E501 some pre-configured values. @@ -100,6 +104,7 @@ def _read_config_file(cf, _globals=globals(), _locals=locals(), interactive=True def _validate_local(x): + # type: (str) -> bool """Returns whether or not a variable should be imported. Will return False for any default modules (sys), or if they are detected as private vars (starting with a _)""" @@ -109,10 +114,11 @@ def _validate_local(x): DEFAULT_PRESTART_FILE = _probe_config_file(".scapy_prestart.py") DEFAULT_STARTUP_FILE = _probe_config_file(".scapy_startup.py") -SESSION = None +SESSION = {} # type: Dict[str, Any] def _usage(): + # type: () -> None print( "Usage: scapy.py [-s sessionfile] [-c new_startup_file] " "[-p new_prestart_file] [-C] [-P] [-H]\n" @@ -130,6 +136,7 @@ def _usage(): def _load(module, globals_dict=None, symb_list=None): + # type: (str, Optional[Dict[str, Any]], Optional[List[str]]) -> None """Loads a Python module to make variables, objects and functions available globally. @@ -159,6 +166,7 @@ def _load(module, globals_dict=None, symb_list=None): def load_module(name, globals_dict=None, symb_list=None): + # type: (str, Optional[Dict[str, Any]], Optional[List[str]]) -> None """Loads a Scapy module to make variables, objects and functions available globally. @@ -168,6 +176,7 @@ def load_module(name, globals_dict=None, symb_list=None): def load_layer(name, globals_dict=None, symb_list=None): + # type: (str, Optional[Dict[str, Any]], Optional[List[str]]) -> None """Loads a Scapy layer module to make variables, objects and functions available globally. @@ -177,6 +186,7 @@ def load_layer(name, globals_dict=None, symb_list=None): def load_contrib(name, globals_dict=None, symb_list=None): + # type: (str, Optional[Dict[str, Any]], Optional[List[str]]) -> None """Loads a Scapy contrib module to make variables, objects and functions available globally. @@ -197,7 +207,11 @@ def load_contrib(name, globals_dict=None, symb_list=None): raise e # Let's raise the original error to avoid confusion -def list_contrib(name=None, ret=False, _debug=False): +def list_contrib(name=None, # type: Optional[str] + ret=False, # type: bool + _debug=False # type: bool + ): + # type: (...) -> Optional[List[Dict[str, Union[str, None]]]] """Show the list of all existing contribs. :param name: filter to search the contribs @@ -216,7 +230,7 @@ def list_contrib(name=None, ret=False, _debug=False): name = "*.py" elif "*" not in name and "?" not in name and not name.endswith(".py"): name += ".py" - results = [] + results = [] # type: List[Dict[str, Union[str, None]]] dir_path = os.path.join(os.path.dirname(__file__), "contrib") if sys.version_info >= (3, 5): name = os.path.join(dir_path, "**", name) @@ -259,6 +273,7 @@ def list_contrib(name=None, ret=False, _debug=False): else: for desc in results: print("%(name)-20s: %(description)-40s status=%(status)s" % desc) + return None ############################## @@ -266,15 +281,17 @@ def list_contrib(name=None, ret=False, _debug=False): ############################## def update_ipython_session(session): + # type: (Dict[str, Any]) -> None """Updates IPython session with a custom one""" try: - global get_ipython + from IPython import get_ipython get_ipython().user_ns.update(session) except Exception: pass -def save_session(fname=None, session=None, pickleProto=-1): +def save_session(fname="", session=None, pickleProto=-1): + # type: (str, Optional[Dict[str, Any]], int) -> None """Save current Scapy session to the file specified in the fname arg. params: @@ -283,19 +300,20 @@ def save_session(fname=None, session=None, pickleProto=-1): - pickleProto: pickle proto version (default: -1 = latest)""" from scapy import utils from scapy.config import conf, ConfClass - if fname is None: + if not fname: fname = conf.session if not fname: conf.session = fname = utils.get_temp_file(keep=True) log_interactive.info("Use [%s] as session file" % fname) - if session is None: + if not session: try: + from IPython import get_ipython session = get_ipython().user_ns except Exception: session = six.moves.builtins.__dict__["scapy_session"] - to_be_saved = session.copy() + to_be_saved = cast(Dict[str, Any], session).copy() if "__builtins__" in to_be_saved: del(to_be_saved["__builtins__"]) @@ -322,6 +340,7 @@ def save_session(fname=None, session=None, pickleProto=-1): def load_session(fname=None): + # type: (Optional[Union[str, None]]) -> None """Load current Scapy session from the file specified in the fname arg. This will erase any existing session. @@ -348,6 +367,7 @@ def load_session(fname=None): def update_session(fname=None): + # type: (Optional[Union[str, None]]) -> None """Update current Scapy session from the file specified in the fname arg. params: @@ -364,7 +384,10 @@ def update_session(fname=None): update_ipython_session(scapy_session) -def init_session(session_name, mydict=None): +def init_session(session_name, # type: Optional[Union[str, None]] + mydict=None # type: Optional[Union[Dict[str, Any], None]] + ): + # type: (...) -> None from scapy.config import conf global SESSION global GLOBKEYS @@ -373,7 +396,6 @@ def init_session(session_name, mydict=None): six.moves.builtins.__dict__.update(scapy_builtins) GLOBKEYS.extend(scapy_builtins) GLOBKEYS.append("scapy_session") - scapy_builtins = None if session_name: try: @@ -418,6 +440,7 @@ def init_session(session_name, mydict=None): def scapy_delete_temp_files(): + # type: () -> None from scapy.config import conf for f in conf.temp_files: try: @@ -428,20 +451,22 @@ def scapy_delete_temp_files(): def _prepare_quote(quote, author, max_len=78): + # type: (str, str, int) -> List[str] """This function processes a quote and returns a string that is ready to be used in the fancy prompt. """ - quote = quote.split(' ') + _quote = quote.split(' ') max_len -= 6 lines = [] - cur_line = [] + cur_line = [] # type: List[str] def _len(line): + # type: (List[str]) -> int return sum(len(elt) for elt in line) + len(line) - 1 - while quote: - if not cur_line or (_len(cur_line) + len(quote[0]) - 1 <= max_len): - cur_line.append(quote.pop(0)) + while _quote: + if not cur_line or (_len(cur_line) + len(_quote[0]) - 1 <= max_len): + cur_line.append(_quote.pop(0)) continue lines.append(' | %s' % ' '.join(cur_line)) cur_line = [] @@ -453,6 +478,7 @@ def _len(line): def interact(mydict=None, argv=None, mybanner=None, loglevel=logging.INFO): + # type: (Optional[Any], Optional[Any], Optional[Any], int) -> None """Starts Scapy's console.""" global SESSION global GLOBKEYS @@ -618,7 +644,7 @@ def interact(mydict=None, argv=None, mybanner=None, loglevel=logging.INFO): quote, author = choice(QUOTES) the_banner.extend(_prepare_quote(quote, author, max_len=39)) the_banner.append(" |") - the_banner = "\n".join( + banner_text = "\n".join( logo + banner for logo, banner in six.moves.zip_longest( (conf.color_theme.logo(line) for line in the_logo), (conf.color_theme.success(line) for line in the_banner), @@ -626,13 +652,13 @@ def interact(mydict=None, argv=None, mybanner=None, loglevel=logging.INFO): ) ) else: - the_banner = "Welcome to Scapy (%s)" % conf.version + banner_text = "Welcome to Scapy (%s)" % conf.version if mybanner is not None: - the_banner += "\n" - the_banner += mybanner + banner_text += "\n" + banner_text += mybanner if IPYTHON: - banner = the_banner + " using IPython %s\n" % IPython.__version__ + banner = banner_text + " using IPython %s\n" % IPython.__version__ try: from traitlets.config.loader import Config except ImportError: @@ -647,12 +673,14 @@ def interact(mydict=None, argv=None, mybanner=None, loglevel=logging.INFO): exec_lines=["print(\"\"\"" + banner + "\"\"\")"] ) except Exception: - code.interact(banner=the_banner, local=SESSION) + code.interact(banner=banner_text, local=SESSION) else: cfg = Config() try: - get_ipython - except NameError: + from IPython import get_ipython + if not get_ipython(): + raise ImportError + except ImportError: # Set "classic" prompt style when launched from run_scapy(.bat) files # noqa: E501 # Register and apply scapy color+prompt style apply_ipython_style(shell=cfg.TerminalInteractiveShell) @@ -668,9 +696,9 @@ def interact(mydict=None, argv=None, mybanner=None, loglevel=logging.INFO): try: start_ipython(config=cfg, user_ns=SESSION) except (AttributeError, TypeError): - code.interact(banner=the_banner, local=SESSION) + code.interact(banner=banner_text, local=SESSION) else: - code.interact(banner=the_banner, local=SESSION) + code.interact(banner=banner_text, local=SESSION) if conf.session: save_session(conf.session, SESSION) diff --git a/scapy/tools/UTscapy.py b/scapy/tools/UTscapy.py index 31971c2410d..dfba41b25fc 100644 --- a/scapy/tools/UTscapy.py +++ b/scapy/tools/UTscapy.py @@ -29,7 +29,6 @@ from scapy.modules.six.moves import range from scapy.compat import base64_bytes, bytes_hex, plain_str - # Util class # class Bunch: @@ -696,7 +695,7 @@ def usage(): -T\t\t: if -t is used with *, remove a specific file (can be used many times) -l\t\t: generate local .js and .css files -F\t\t: expand only failed tests --b\t\t: stop at first failed campaign +-b\t\t: don't stop at the first failed campaign -d\t\t: dump campaign -D\t\t: dump campaign and stop -C\t\t: don't calculate CRC and SHA @@ -704,6 +703,7 @@ def usage(): -c\t\t: load a .utsc config file -q\t\t: quiet mode -qq\t\t: [silent mode] +-x\t\t: use pyannotate -n \t: only tests whose numbers are given (eg. 1,3-7,12) -m \t: additional module to put in the namespace -k ,,...\t: include only tests with one of those keywords (can be used many times) @@ -801,13 +801,14 @@ def main(): SCAPY = "scapy" MODULES = [] TESTFILES = [] + ANNOTATIONS_MODE = False try: - opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DdCFqP:s:") + opts = getopt.getopt(argv, "o:t:T:c:f:hbln:m:k:K:DdCFqP:s:x") for opt, optarg in opts[0]: if opt == "-h": usage() elif opt == "-b": - BREAKFAILED = True + BREAKFAILED = False elif opt == "-F": ONLYFAILED = True elif opt == "-q": @@ -818,6 +819,8 @@ def main(): DUMP = 1 elif opt == "-C": CRC = False + elif opt == "-x": + ANNOTATIONS_MODE = True elif opt == "-s": SCAPY = optarg elif opt == "-P": @@ -885,6 +888,14 @@ def main(): if six.PY2: KW_KO.append("python3_only") + if ANNOTATIONS_MODE: + try: + from pyannotate_runtime import collect_types + except ImportError: + raise ImportError("Please install pyannotate !") + collect_types.init_types_collection() + collect_types.start() + if VERB > 2: print("### Booting scapy...", file=sys.stderr) try: @@ -964,6 +975,11 @@ def main(): if VERB > 2: print("### Writing output...", file=sys.stderr) + + if ANNOTATIONS_MODE: + collect_types.stop() + collect_types.dump_stats("pyannotate_results") + # Concenate outputs if FORMAT == Format.HTML: glob_output = pack_html_campaigns(runned_campaigns, glob_output, LOCAL, glob_title) diff --git a/scapy/tools/scapy_pyannotate.py b/scapy/tools/scapy_pyannotate.py new file mode 100644 index 00000000000..f7d12825a29 --- /dev/null +++ b/scapy/tools/scapy_pyannotate.py @@ -0,0 +1,21 @@ +# This file is part of Scapy +# See http://www.secdev.org/projects/scapy for more information +# Copyright (C) Philippe Biondi +# This program is published under a GPLv2 license + +""" +Wrap Scapy's shell in pyannotate. +""" + +import os +import sys +sys.path.insert(0, os.path.abspath('../../')) + +from pyannotate_runtime import collect_types # noqa: E402 +from scapy.main import interact # noqa: E402 + +collect_types.init_types_collection() +with collect_types.collect(): + interact() + +collect_types.dump_stats("pyannotate_results_main") diff --git a/test/regression.uts b/test/regression.uts index 09480642e5e..cd6af9db9b3 100644 --- a/test/regression.uts +++ b/test/regression.uts @@ -747,7 +747,8 @@ r assert r in [ b'u\x14\x00\x1c\xc2\xf6\x80\x00\xde\x01k\xd3\x7f\x00\x00\x01\x7f\x00\x00\x01y\xc9>\xa6\x84\xd8\xc2\xb7', b'E\xa7\x00\x1c\xb0c\xc0\x00\xf6\x01U\xd3\x7f\x00\x00\x01\x7f\x00\x00\x01\xfex\xb3\x92B<\x0b\xb8', - '\x85\x7f\x00\x1c\xc2\xf6\x00\x00\xde\x01\xdbh\x7f\x00\x00\x01\x7f\x00\x00\x01y\xc9>\xa6\x84\xd8\xc2\xb7' + b'\x85\x7f\x00\x1c\xc2\xf6\x00\x00\xde\x01\xdbh\x7f\x00\x00\x01\x7f\x00\x00\x01y\xc9>\xa6\x84\xd8\xc2\xb7', + b'\xf5H\x00\x1c\x0fr \x00\xff\x01\xde#\x7f\x00\x00\x01\x7f\x00\x00\x01\xfex\xb3\x92B<\x0b\xb8' ] = fuzz a Packet with MultipleTypeField @@ -3679,7 +3680,8 @@ random.seed(0x2807) p = IPv6()/ICMPv6NIQueryIPv4(data="freebsd") h_py2 = '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:y\xb2V<\x7f\x87\x14\xde' h_py3 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:\x88\xccb\x19~\x9e\xe3a' -assert p.hashret() in [h_py2, h_py3] +h_py3_2 = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00:$#\xb5\xb7\xd0\xbf \xe2' +assert p.hashret() in [h_py2, h_py3, h_py3_2] ############ diff --git a/test/run_tests_py2.bat b/test/run_tests_py2.bat index 2eeaeab9de4..385b2f8eacd 100644 --- a/test/run_tests_py2.bat +++ b/test/run_tests_py2.bat @@ -5,7 +5,7 @@ set PWD=%MYDIR% set PYTHONPATH=%MYDIR% set PYTHONDONTWRITEBYTECODE=True if [%1]==[] ( - python "%MYDIR%\scapy\tools\UTscapy.py" -c configs\\windows2.utsc -o scapy_regression_test_%date:~6,4%_%date:~3,2%_%date:~0,2%.html + python "%MYDIR%\scapy\tools\UTscapy.py" -c configs\\windows2.utsc -b -o scapy_regression_test_%date:~6,4%_%date:~3,2%_%date:~0,2%.html ) else ( python "%MYDIR%\scapy\tools\UTscapy.py" %* ) diff --git a/tox.ini b/tox.ini index d34f0784b30..d5365cf5002 100644 --- a/tox.ini +++ b/tox.ini @@ -111,6 +111,7 @@ commands = codecov -e TOXENV [testenv:generate_docs] +description = "Regenerates the API reference doc tree" skip_install = true changedir = doc/scapy deps = sphinx @@ -120,6 +121,14 @@ commands = sphinx-build -b html . _build/html +[testenv:mypy] +description = "Check Scapy compliance against static typing" +skip_install = true +deps = mypy + typing +commands = python .config/mypy/mypy_check.py + + [testenv:docs] skip_install = true changedir = doc/scapy @@ -134,7 +143,7 @@ description = "Check code for Grammar mistakes" skip_install = true deps = codespell # inet6, dhcp6 and the ipynb files contains french: ignore them -commands = codespell --ignore-words=.travis/codespell_ignore.txt --skip="*.pyc,*.png,*.jpg,*.ods,*.raw,*.pdf,*.pcap,*.js,*.html,*.der,*_build*,*inet6.py,*dhcp6.py,*.ipynb,*.svg,*.gif,*.obs" scapy/ doc/ test/ .github/ +commands = codespell --ignore-words=.config/codespell_ignore.txt --skip="*.pyc,*.png,*.jpg,*.ods,*.raw,*.pdf,*.pcap,*.js,*.html,*.der,*_build*,*inet6.py,*dhcp6.py,*.ipynb,*.svg,*.gif,*.obs" scapy/ doc/ test/ .github/ [testenv:twine] From 790ee129028740a396ef6c4e56855d997d223987 Mon Sep 17 00:00:00 2001 From: gpotter2 Date: Thu, 25 Jul 2019 15:49:41 +0200 Subject: [PATCH 2/2] Apply guedou's suggestion --- .config/mypy/mypy.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.config/mypy/mypy.ini b/.config/mypy/mypy.ini index 32bfce7fa5e..738881bb00a 100644 --- a/.config/mypy/mypy.ini +++ b/.config/mypy/mypy.ini @@ -7,4 +7,7 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-traitlets.config.loader] -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = True + +[mypy-scapy.modules.six,scapy.modules.winpcapy] +ignore_errors = True