Bundle standalone 'manufdb' file to comply with Wireshark changes.

Loading the dictionnary takes 0.5s by itsef, so we have to cache it in
order to keep reasonable boot times.

scapy/arch/windows/__init__.py

@@ -38,7 +38,7 @@
 from scapy.pton_ntop import inet_ntop, inet_pton
 from scapy.utils import atol, itom, mac2str, str2mac
 from scapy.utils6 import construct_source_candidate_set, in6_getscope
-from scapy.data import ARPHDR_ETHER, load_manuf
+from scapy.data import ARPHDR_ETHER
 from scapy.compat import plain_str
 from scapy.supersocket import SuperSocket
 
@@ -202,20 +202,6 @@ def _reload(self):
         )
         self.cmd = win_find_exe("cmd", installsubdir="System32",
                                 env="SystemRoot")
-        if self.wireshark:
-            try:
-                new_manuf = load_manuf(
-                    os.path.sep.join(
-                        self.wireshark.split(os.path.sep)[:-1]
-                    ) + os.path.sep + "manuf"
-                )
-            except (IOError, OSError):  # FileNotFoundError not available on Py2 - using OSError  # noqa: E501
-                log_loading.warning("Wireshark is installed, but cannot read manuf !")  # noqa: E501
-                new_manuf = None
-            if new_manuf:
-                # Inject new ManufDB
-                conf.manufdb.__dict__.clear()
-                conf.manufdb.__dict__.update(new_manuf.__dict__)
 
 
 def _exec_cmd(command):

scapy/config.py

@@ -984,6 +984,11 @@ class Conf(ConfClass):
     #: manipulate it
     route6 = None  # type: 'scapy.route6.Route6'
     manufdb = None  # type: 'scapy.data.ManufDA'
+    ethertypes = None  # type: 'scapy.data.EtherDA'
+    protocols = None  # type: 'scapy.dadict.DADict[int, str]'
+    services_udp = None  # type: 'scapy.dadict.DADict[int, str]'
+    services_tcp = None  # type: 'scapy.dadict.DADict[int, str]'
+    services_sctp = None  # type: 'scapy.dadict.DADict[int, str]'
     # 'route6' will be filed by route6.py
     teredoPrefix = ""  # type: str
     teredoServerPort = None  # type: int

scapy/dadict.py

@@ -10,15 +10,20 @@
 from scapy.error import Scapy_Exception
 from scapy.compat import plain_str
 
+# Typing
 from typing import (
     Any,
     Dict,
     Generic,
     Iterator,
     List,
+    Tuple,
+    Type,
     TypeVar,
     Union,
 )
+from scapy.compat import Self
+
 
 ###############################
 #  Direct Access dictionary   #
@@ -66,11 +71,13 @@ class DADict(Generic[_K, _V]):
 
         ETHER_TYPES.IPv4 -> 2048
     """
+    __slots__ = ["_name", "d"]
+
     def __init__(self, _name="DADict", **kargs):
         # type: (str, **Any) -> None
         self._name = _name
         self.d = {}  # type: Dict[_K, _V]
-        self.update(kargs)
+        self.update(kargs)  # type: ignore
 
     def ident(self, v):
         # type: (_V) -> str
@@ -82,7 +89,7 @@ def ident(self, v):
         return "unknown"
 
     def update(self, *args, **kwargs):
-        # type: (*Dict[str, _V], **Dict[str, _V]) -> None
+        # type: (*Dict[_K, _V], **Dict[_K, _V]) -> None
         for k, v in dict(*args, **kwargs).items():
             self[k] = v  # type: ignore
 
@@ -148,3 +155,12 @@ def __getattr__(self, attr):
     def __dir__(self):
         # type: () -> List[str]
         return [self.ident(x) for x in self.itervalues()]
+
+    def __reduce__(self):
+        # type: () -> Tuple[Type[Self], Tuple[str], Tuple[Dict[_K, _V]]]
+        return (self.__class__, (self._name,), (self.d,))
+
+    def __setstate__(self, state):
+        # type: (Tuple[Dict[_K, _V]]) -> Self
+        self.d.update(state[0])
+        return self

scapy/data.py

@@ -9,14 +9,15 @@
 
 import calendar
 import os
+import pickle
 import warnings
 
 
 from scapy.dadict import DADict, fixname
 from scapy.consts import FREEBSD, NETBSD, OPENBSD, WINDOWS
 from scapy.error import log_loading
-from scapy.compat import plain_str
 
+# Typing imports
 from typing import (
     Any,
     Callable,
@@ -28,6 +29,7 @@
     Union,
     cast,
 )
+from scapy.compat import DecoratorCallable
 
 
 ############
@@ -290,17 +292,69 @@
 }
 
 
+def scapy_data_cache(name):
+    # type: (str) -> Callable[[DecoratorCallable], DecoratorCallable]
+    """
+    This decorator caches the loading of 'data' dictionaries, in order to reduce
+    loading times.
+    """
+    from scapy.main import SCAPY_CONFIG_FOLDER
+    if SCAPY_CONFIG_FOLDER is None:
+        # Cannot cache.
+        return lambda x: x
+    cachepath = SCAPY_CONFIG_FOLDER / "cache" / name
+
+    def _cached_loader(func, name=name):
+        # type: (DecoratorCallable, str) -> DecoratorCallable
+        def load(filename=None):
+            # type: (Optional[str]) -> Any
+            cache_id = hash(filename)
+            if cachepath.exists():
+                try:
+                    with cachepath.open("rb") as fd:
+                        data = pickle.load(fd)
+                    if data["id"] == cache_id:
+                        return data["content"]
+                except Exception:
+                    log_loading.warning(
+                        "Couldn't load cache from %s" % str(cachepath),
+                        exc_info=True,
+                    )
+                    cachepath.unlink()
+            # Cache does not exist or is invalid.
+            content = func(filename)
+            data = {
+                "content": content,
+                "id": cache_id,
+            }
+            try:
+                cachepath.parent.mkdir(parents=True, exist_ok=True)
+                with cachepath.open("wb") as fd:
+                    pickle.dump(data, fd)
+                return content
+            except Exception:
+                log_loading.warning(
+                    "Couldn't cache %s into %s" % (repr(func), str(cachepath)),
+                    exc_info=True,
+                )
+                return content
+        return load  # type: ignore
+    return _cached_loader
+
+
 def load_protocols(filename, _fallback=None, _integer_base=10,
                    _cls=DADict[int, str]):
-    # type: (str, Optional[bytes], int, type) -> DADict[int, str]
-    """"Parse /etc/protocols and return values as a dictionary."""
+    # type: (str, Optional[str], int, type) -> DADict[int, str]
+    """"
+    Parse /etc/protocols and return values as a dictionary.
+    """
     dct = _cls(_name=filename)  # type: DADict[int, str]
 
     def _process_data(fdesc):
-        # type: (Iterator[bytes]) -> None
+        # type: (Iterator[str]) -> None
         for line in fdesc:
             try:
-                shrp = line.find(b"#")
+                shrp = line.find("#")
                 if shrp >= 0:
                     line = line[:shrp]
                 line = line.strip()
@@ -320,11 +374,11 @@ def _process_data(fdesc):
     try:
         if not filename:
             raise IOError
-        with open(filename, "rb") as fdesc:
+        with open(filename, "r", errors="backslashreplace") as fdesc:
             _process_data(fdesc)
     except IOError:
         if _fallback:
-            _process_data(iter(_fallback.split(b"\n")))
+            _process_data(iter(_fallback.split("\n")))
         else:
             log_loading.info("Can't open %s file", filename)
     return dct
@@ -354,18 +408,20 @@ def __getitem__(self, attr):
         return super(EtherDA, self).__getitem__(attr)
 
 
-def load_ethertypes(filename):
+@scapy_data_cache("ethertypes")
+def load_ethertypes(filename=None):
     # type: (Optional[str]) -> EtherDA
     """"Parse /etc/ethertypes and return values as a dictionary.
     If unavailable, use the copy bundled with Scapy."""
     from scapy.libs.ethertypes import DATA
-    prot = load_protocols(filename or "Scapy's backup ETHER_TYPES",
+    prot = load_protocols(filename or "<scapy>/ethertypes",
                           _fallback=DATA,
                           _integer_base=16,
                           _cls=EtherDA)
     return cast(EtherDA, prot)
 
 
+@scapy_data_cache("services")
 def load_services(filename):
     # type: (str) -> Tuple[DADict[int, str], DADict[int, str], DADict[int, str]]  # noqa: E501
     tdct = DADict(_name="%s-tcp" % filename)  # type: DADict[int, str]
@@ -472,30 +528,42 @@ def __dir__(self):
         ] + super(ManufDA, self).__dir__()
 
 
-def load_manuf(filename):
-    # type: (str) -> ManufDA
+@scapy_data_cache("manufdb")
+def load_manuf(filename=None):
+    # type: (Optional[str]) -> ManufDA
     """
     Loads manuf file from Wireshark.
 
     :param filename: the file to load the manuf file from
     :returns: a ManufDA filled object
     """
-    manufdb = ManufDA(_name=filename)
-    with open(filename, "rb") as fdesc:
+    manufdb = ManufDA(_name=filename or "<scapy>/manufdb")
+
+    def _process_data(fdesc):
+        # type: (Iterator[str]) -> None
         for line in fdesc:
             try:
                 line = line.strip()
-                if not line or line.startswith(b"#"):
+                if not line or line.startswith("#"):
                     continue
                 parts = line.split(None, 2)
-                ouib, shrt = parts[:2]
-                lng = parts[2].lstrip(b"#").strip() if len(parts) > 2 else b""
+                oui, shrt = parts[:2]
+                lng = parts[2].lstrip("#").strip() if len(parts) > 2 else ""
                 lng = lng or shrt
-                oui = plain_str(ouib)
-                manufdb[oui] = plain_str(shrt), plain_str(lng)
+                manufdb[oui] = shrt, lng
             except Exception:
                 log_loading.warning("Couldn't parse one line from [%s] [%r]",
                                     filename, line, exc_info=True)
+
+    try:
+        if not filename:
+            raise IOError
+        with open(filename, "r", errors="backslashreplace") as fdesc:
+            _process_data(fdesc)
+    except IOError:
+        # Fallback. Lazy loaded as the file is big.
+        from scapy.libs.manuf import DATA
+        _process_data(iter(DATA.split("\n")))
     return manufdb
 
 
@@ -524,24 +592,19 @@ def select_path(directories, filename):
         "etc",
         "services",
     ))
-    # Default values, will be updated by arch.windows
-    ETHER_TYPES = load_ethertypes(None)
-    MANUFDB = ManufDA()
+    ETHER_TYPES = load_ethertypes()
+    MANUFDB = load_manuf()
 else:
     IP_PROTOS = load_protocols("/etc/protocols")
-    ETHER_TYPES = load_ethertypes("/etc/ethertypes")
     TCP_SERVICES, UDP_SERVICES, SCTP_SERVICES = load_services("/etc/services")
-    MANUFDB = ManufDA()
-    manuf_path = select_path(
-        ['/usr', '/usr/local', '/opt', '/opt/wireshark',
-         '/Applications/Wireshark.app/Contents/Resources'],
-        "share/wireshark/manuf"
+    ETHER_TYPES = load_ethertypes("/etc/ethertypes")
+    MANUFDB = load_manuf(
+        select_path(
+            ['/usr', '/usr/local', '/opt', '/opt/wireshark',
+             '/Applications/Wireshark.app/Contents/Resources'],
+            "share/wireshark/manuf"
+        )
     )
-    if manuf_path:
-        try:
-            MANUFDB = load_manuf(manuf_path)
-        except (IOError, OSError):
-            log_loading.warning("Cannot read wireshark manuf database")
 
 
 #####################

scapy/libs/ethertypes.py

@@ -39,7 +39,7 @@
 # scapy/tools/generate_ethertypes.py
 # based on OpenBSD public source.
 
-DATA = b"""
+DATA = """
 #
 # Ethernet frame types
 #       This file describes some of the various Ethernet