Bump mypy from 0.812 to 0.931 in /tools (#33082)

Changes to adapt:

 * Deals with changes in how mypy handles metaclasses

 * Prefers sys.platform == "win32" due to
   python/mypy#8166 and mypy not having
   WindowsError defined by default any more

 * Installs various typestubs

 * Rewrites tox.ini to avoid duplicating everything, and allowing new
   versions of Python to be easily tested (as tox -e py310-mypy will
   now work without further changes).

 * Make mypy warn when it thinks code is unreachable.

Co-authored-by: Sam Sneddon <[email protected]>
Co-authored-by: Philip Jägenstedt <[email protected]>

tools/gitignore/gitignore.py

@@ -27,7 +27,7 @@
 def fnmatch_translate(pat):
     # type: (bytes) -> Tuple[bool, Pattern[bytes]]
     parts = []
-    seq = None
+    seq = None  # type: Optional[int]
     i = 0
     any_char = b"[^/]"
     if pat[0:1] == b"/":
@@ -60,10 +60,10 @@ def fnmatch_translate(pat):
             # TODO: this doesn't really handle invalid sequences in the right way
             if c == b"]":
                 seq = None
-                if parts[-1:] == b"[":
+                if parts[-1] == b"[":
                     parts = parts[:-1]
-                elif parts[-1:] == b"^" and parts[-2:-1] == b"[":
-                    parts = parts[:-2]
+                elif parts[-1] == b"^" and parts[-2] == b"[":
+                    raise ValueError
                 else:
                     parts.append(c)
             elif c == b"-":

tools/gitignore/tests/test_gitignore.py

@@ -26,6 +26,7 @@
     (b"a?c", True, [b"abc"]),
     (b"a[^b]c", True, [b"acc"]),
     (b"a[b-c]c", True, [b"abc", b"acc"]),
+    (b"a[]c", True, [b"ac"]),
 ]  # type: Sequence[Tuple[bytes, bool, Iterable[bytes]]]
 
 mismatch_data = [

tools/manifest/item.py

@@ -8,16 +8,7 @@
 MYPY = False
 if MYPY:
     # MYPY is set to True when run under Mypy.
-    from typing import Optional
-    from typing import Text
-    from typing import Dict
-    from typing import Tuple
-    from typing import List
-    from typing import Union
-    from typing import Type
-    from typing import Any
-    from typing import Sequence
-    from typing import Hashable
+    from typing import Any, Dict, Hashable, List, Optional, Sequence, Text, Tuple, Type, Union, cast
     from .manifest import Manifest
     Fuzzy = Dict[Optional[Tuple[Text, Text, Text]], List[int]]
     PageRanges = Dict[Text, List[int]]
@@ -31,14 +22,23 @@ class ManifestItemMeta(ABCMeta):
     attribute, and otherwise behaves like an ABCMeta."""
 
     def __new__(cls, name, bases, attrs):
-        # type: (Type[ManifestItemMeta], str, Tuple[ManifestItemMeta, ...], Dict[str, Any]) -> ManifestItemMeta
-        rv = super(ManifestItemMeta, cls).__new__(cls, name, bases, attrs)
-        if not isabstract(rv):
-            assert issubclass(rv, ManifestItem)
-            assert isinstance(rv.item_type, str)
-            item_types[rv.item_type] = rv
-
-        return rv  # type: ignore
+        # type: (Type[ManifestItemMeta], str, Tuple[type], Dict[str, Any]) -> ManifestItemMeta
+        inst = super(ManifestItemMeta, cls).__new__(cls, name, bases, attrs)
+        if isabstract(inst):
+            return inst
+
+        assert issubclass(inst, ManifestItem)
+        if MYPY:
+            inst_ = cast(Type[ManifestItem], inst)
+            item_type = cast(str, inst_.item_type)
+        else:
+            inst_ = inst
+            assert isinstance(inst_.item_type, str)
+            item_type = inst_.item_type
+
+        item_types[item_type] = inst_
+
+        return inst_
 
 
 class ManifestItem(metaclass=ManifestItemMeta):

tools/manifest/sourcefile.py

@@ -174,7 +174,9 @@ def _parse_html(f):
     doc = html5lib.parse(f, treebuilder="etree", useChardet=False)
     if MYPY:
         return cast(ElementTree.Element, doc)
-    return doc
+    else:
+        # (needs to be in else for mypy to believe this is reachable)
+        return doc
 
 def _parse_xml(f):
     # type: (BinaryIO) -> ElementTree.Element

tools/manifest/typedata.py

@@ -146,8 +146,8 @@ def __delitem__(self, key):
     def __iter__(self):
         # type: () -> Iterator[Tuple[Text, ...]]
         """Iterator over keys in the TypeData in codepoint order"""
-        data_node = self._data  # type: Optional[Dict[Text, Any]]
-        json_node = self._json_data  # type: Optional[Dict[Text, Any]]
+        data_node = self._data  # type: Optional[Union[Dict[Text, Any], Set[item.ManifestItem]]]
+        json_node = self._json_data  # type: Optional[Union[Dict[Text, Any], List[Any]]]
         path = tuple()  # type: Tuple[Text, ...]
         stack = [(data_node, json_node, path)]
         while stack:
@@ -174,21 +174,21 @@ def __len__(self):
         # type: () -> int
         count = 0
 
-        stack = [self._data]
+        stack = [self._data]  # type: List[Union[Dict[Text, Any], Set[item.ManifestItem]]]
         while stack:
             v = stack.pop()
             if isinstance(v, set):
                 count += 1
             else:
                 stack.extend(v.values())
 
-        stack = [self._json_data]
-        while stack:
-            v = stack.pop()
-            if isinstance(v, list):
+        json_stack = [self._json_data]  # type: List[Union[Dict[Text, Any], List[Any]]]
+        while json_stack:
+            json_v = json_stack.pop()
+            if isinstance(json_v, list):
                 count += 1
             else:
-                stack.extend(v.values())
+                json_stack.extend(json_v.values())
 
         return count
 

tools/manifest/utils.py

@@ -1,6 +1,6 @@
 import os
-import platform
 import subprocess
+import sys
 
 MYPY = False
 if MYPY:
@@ -31,7 +31,7 @@ def rel_path_to_url(rel_path, url_base="/"):
 
 def from_os_path(path):
     # type: (Text) -> Text
-    assert os.path.sep == u"/" or platform.system() == "Windows"
+    assert os.path.sep == u"/" or sys.platform == "win32"
     if u"/" == os.path.sep:
         rv = path
     else:
@@ -43,7 +43,7 @@ def from_os_path(path):
 
 def to_os_path(path):
     # type: (Text) -> Text
-    assert os.path.sep == u"/" or platform.system() == "Windows"
+    assert os.path.sep == u"/" or sys.platform == "win32"
     if u"\\" in path:
         raise ValueError("normalised path contains \\")
     if u"/" == os.path.sep:
@@ -59,7 +59,7 @@ def gitfunc(cmd, *args):
         try:
             return subprocess.check_output(full_cmd, cwd=path, stderr=subprocess.STDOUT).decode('utf8')
         except Exception as e:
-            if platform.uname()[0] == "Windows" and isinstance(e, WindowsError):
+            if sys.platform == "win32" and isinstance(e, WindowsError):
                 full_cmd[0] = u"git.bat"
                 return subprocess.check_output(full_cmd, cwd=path, stderr=subprocess.STDOUT).decode('utf8')
             else:

tools/manifest/vcs.py

@@ -102,10 +102,7 @@ def __init__(self, tests_root, url_base, cache_path, manifest_path=None, rebuild
                                                 extras=[b".git/"],
                                                 cache=self.ignore_cache)
         git = GitHasher(tests_root)
-        if git is not None:
-            self.hash_cache = git.hash_cache()
-        else:
-            self.hash_cache = {}
+        self.hash_cache = git.hash_cache()
 
     def __iter__(self):
         # type: () -> Iterator[Tuple[Text, Optional[Text], bool]]

tools/mypy.ini

@@ -21,6 +21,9 @@ warn_redundant_casts = True
 warn_return_any = True
 warn_unused_configs = True
 warn_unused_ignores = True
+warn_unreachable = True
+
+show_error_codes = True
 
 # Ignore missing or untyped libraries.
 

tools/requirements_mypy.txt

@@ -1,3 +1,12 @@
-mypy==0.812
+mypy==0.931
 mypy-extensions==0.4.3
+toml==0.10.2
 typed-ast==1.4.3
+types-atomicwrites==1.4.1
+types-python-dateutil==2.8.9
+types-PyYAML==6.0.0
+types-requests==2.25.11
+types-setuptools==57.4.2
+types-six==1.16.2
+types-ujson==4.2.0
+typing-extensions==3.10.0.2

tools/tox.ini

@@ -1,52 +1,24 @@
 [tox]
-envlist = py36,py37,py38,py39,{py36,py37,py38,py39}-flake8,{py36,py37,py38,py39}-mypy
+envlist = py36,py37,py38,py39,{py36,py37,py38,py39}-{flake8,mypy}
 skipsdist=True
-skip_missing_interpreters = False
+skip_missing_interpreters=False
 
 [testenv]
 deps =
-  -r{toxinidir}/requirements_pytest.txt
-  -r{toxinidir}/requirements_tests.txt
+  !flake8-!mypy: -r{toxinidir}/requirements_pytest.txt
+  !flake8-!mypy: -r{toxinidir}/requirements_tests.txt
+  flake8: -r{toxinidir}/requirements_flake8.txt
+  mypy: -r{toxinidir}/requirements_mypy.txt
 
-commands = pytest --cov=tools --cov-report=term {posargs}
+changedir =
+  mypy: {toxinidir}/..
+
+commands =
+  !flake8-!mypy: pytest --cov=tools --cov-report=term {posargs}
+  flake8: flake8 --append-config={toxinidir}/flake8.ini {posargs}
+  mypy: mypy --config-file={toxinidir}/mypy.ini tools/
 
 passenv =
   HYPOTHESIS_PROFILE
   PY_COLORS
   TASKCLUSTER_ROOT_URL
-
-[testenv:py36-flake8]
-deps = -rrequirements_flake8.txt
-commands = flake8 --append-config={toxinidir}/flake8.ini {posargs}
-
-[testenv:py37-flake8]
-deps = -rrequirements_flake8.txt
-commands = flake8 --append-config={toxinidir}/flake8.ini {posargs}
-
-[testenv:py38-flake8]
-deps = -rrequirements_flake8.txt
-commands = flake8 --append-config={toxinidir}/flake8.ini {posargs}
-
-[testenv:py39-flake8]
-deps = -rrequirements_flake8.txt
-commands = flake8 --append-config={toxinidir}/flake8.ini {posargs}
-
-[testenv:py36-mypy]
-deps = -rrequirements_mypy.txt
-changedir = {toxinidir}/..
-commands = mypy --config-file={toxinidir}/mypy.ini tools/
-
-[testenv:py37-mypy]
-deps = -rrequirements_mypy.txt
-changedir = {toxinidir}/..
-commands = mypy --config-file={toxinidir}/mypy.ini tools/
-
-[testenv:py38-mypy]
-deps = -rrequirements_mypy.txt
-changedir = {toxinidir}/..
-commands = mypy --config-file={toxinidir}/mypy.ini tools/
-
-[testenv:py39-mypy]
-deps = -rrequirements_mypy.txt
-changedir = {toxinidir}/..
-commands = mypy --config-file={toxinidir}/mypy.ini tools/

tools/webdriver/webdriver/bidi/transport.py

@@ -12,10 +12,12 @@
 def get_running_loop() -> asyncio.AbstractEventLoop:
     if sys.version_info >= (3, 7):
         return asyncio.get_running_loop()
-    # Unlike the above, this will actually create an event loop
-    # if there isn't one; hopefully running tests in Python >= 3.7
-    # will allow us to catch any behaviour difference
-    return asyncio.get_event_loop()
+    else:
+        # Unlike the above, this will actually create an event loop
+        # if there isn't one; hopefully running tests in Python >= 3.7
+        # will allow us to catch any behaviour difference
+        # (Needs to be in else for mypy to believe this is reachable)
+        return asyncio.get_event_loop()
 
 
 class Transport: