Big diff: Use new "|" union syntax

[Akuli]

Done with quick and dirty scripts:

main script

import pathlib
import re


def parse_list(string):
    assert string.startswith("[")
    depth = 1
    parts = []
    part_start = i = 1
    while depth > 0:
        if depth == 1 and string[i:].startswith(','):
            parts.append(string[part_start:i].strip())
            part_start = i+1
        depth += string[i] == '['
        depth -= string[i] == ']'
        i += 1
    if string[part_start:i-1].strip():
        parts.append(string[part_start:i-1].strip())
    assert parts, string[:100]
    assert "" not in parts
    return (parts, i)


assert parse_list("[x] bruh") == (["x"], 3)
assert parse_list("[x, y[z]] bruh") == (["x", "y[z]"], 9)

for path in list(pathlib.Path("stdlib").rglob("*.pyi")) + list(pathlib.Path("stubs").rglob("*.pyi")):
    if "@python2" in path.parts:
        # no real reason to skip, I just don't like python 2
        continue

    print(path)
    content = path.read_text()

    while True:
        try:
            match = next(
                m
                for m in re.finditer(r"^.*(Union|Optional)\[", content, flags=re.MULTILINE)
                if not re.match("[A-Za-z_][A-Za-z0-9_]* = ", m.group(0).lstrip(" "))  # skip type aliases
                and not content[m.end():].startswith("]")
            )
        except StopIteration:
            break

        parts, list_length = parse_list(content[match.end()-1:])
        start = match.start(1)
        end = match.end() - 1 + list_length
        if match.group(1) == "Union":
            content = content[:start] + " | ".join(parts) + content[end:]
        else:
            [p] = parts
            content = content[:start] + f"{p} | None" + content[end:]

    path.write_text(content)

Import cleaner (can mess up comments, needs manual checking)

import pathlib
import re


for path in list(pathlib.Path("stdlib").rglob("*.pyi")) + list(pathlib.Path("stubs").rglob("*.pyi")):
    if "@python2" in path.parts:
        # no real reason to skip, I just don't like python 2
        continue

    print(path)
    content = path.read_text()

    for old in ["Union", "Optional"]:
        if old + "[" not in content:
            content = re.sub(rf", \b{old}\b", "", content)
            content = re.sub(rf"\b{old}\b,", "", content)
            content = content.replace(r"from typing import " + old + "\n", "")
    path.write_text(content)

Files left untouched (e.g. because scripts did something weird):

• All @python2 stubs (stdlib and third party)
• Everything under stubs/protobuf/google/
• stdlib/_ast.pyi
• stdlib/_thread.pyi
• stdlib/ast.pyi
• stdlib/asyncio/tasks.pyi
• stdlib/configparser.pyi
• stdlib/ctypes/__init__.pyi
• stdlib/email/_header_value_parser.pyi
• stdlib/os/__init__.pyi
• stdlib/pickle.pyi
• stdlib/types.pyi
• stdlib/zoneinfo/__init__.pyi
• stubs/Jinja2/jinja2/nodes.pyi
• stubs/click/click/core.pyi
• stubs/requests/requests/sessions.pyi
• stubs/six/six/__init__.pyi
• stubs/typed-ast/typed_ast/ast27.pyi
• stubs/typed-ast/typed_ast/ast3.pyi

[JelleZijlstra]

Thanks, I checked a couple of file and this looks great.

Will leave this open for a while longer in case someone has an objection.

[srittau]

Thanks, I think it's a good idea to have the stubs as modern as can be, as they are likely to be used as examples for other stub creators.

[Akuli]

I think I'll merge this once CI passes. Waiting longer causes conflicts that I have to fix manually, and that's error-prone and hard to review as a part of a huge PR.

[github-actions]

According to mypy_primer, this change has no effect on the checked open source code. 🤖🎉

[levrik]

@Akuli Will this also work on Python 3.9 because as far as I know this new syntax is being introduced in Python 3.10 ?

[Akuli]

The syntax works in .pyi files on Python 3.9 and all other versions we support, even 2.7. If you need it in .py files, you can use from __future__ import annotations on Python 3.7+.

[levrik]

@Akuli Ah, thanks!