From 0c0bcf40038c99776a6f6f3322c91278ea8e568c Mon Sep 17 00:00:00 2001 From: Bruce Merry Date: Mon, 15 Jun 2020 09:10:37 +0200 Subject: [PATCH] Introduce ReadableBuffer and WriteableBuffer Union aliases Since typing doesn't yet have a way to express buffer protocol objects (python/typing#593), various interfaces have ended up with a mish-mash of options: some list just bytes (or just bytearray, when writable), some include mmap, some include memoryview, I think none of them include array.array even though it's explicitly mentioned as bytes-like, etc. I ran into problems because RawIOBase.readinto didn't allow for memoryview. To allow for some uniformity until the fundamental issue is resolved, I've introduced _typeshed.ReadableBuffer and _typeshed.WriteableBuffer, and applied them in stdlib/3/io.pyi as an example. If these get rolled out in more places, it will mean that we have only one place where they have to get tweaked in future, or swapped out for a public protocol. This unfortunately does have the potential to break code that inherits from RawIOBase/BufferedIOBase and overrides these methods, because the base method is now more general and so the override now needs to accept these types as well (which is why I've also updated gzip and lzma). However, it should be a reasonably easy fix, and will make the downstream annotations more correct. I'm not 100% happy with the names: bytes-like is slightly stricter than just buffer protocol (it must be able to export a C-contiguous buffer), but in practice I'd be surprised if there are types for which there is a difference at static analysis time (e.g. not every memoryview instance is bytes-like, but that's a property of instances, not types). --- stdlib/2and3/_typeshed/__init__.pyi | 5 +++++ stdlib/3/gzip.pyi | 4 ++-- stdlib/3/io.pyi | 19 +++++++++---------- stdlib/3/lzma.pyi | 4 ++-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/stdlib/2and3/_typeshed/__init__.pyi b/stdlib/2and3/_typeshed/__init__.pyi index 3f64d2e6ecb1..8ace2787135e 100644 --- a/stdlib/2and3/_typeshed/__init__.pyi +++ b/stdlib/2and3/_typeshed/__init__.pyi @@ -12,6 +12,8 @@ # If on Python versions < 3.10 and "from __future__ import annotations" # is not used, types from this module must be quoted. +import array +import mmap import sys from typing import Protocol, Text, TypeVar, Union from typing_extensions import Literal @@ -67,3 +69,6 @@ class SupportsReadline(Protocol[_T_co]): def readline(self, __length: int = ...) -> _T_co: ... class SupportsWrite(Protocol[_T_contra]): def write(self, __s: _T_contra) -> int: ... + +ReadableBuffer = Union[bytes, bytearray, memoryview, array.array, mmap.mmap] +WriteableBuffer = Union[bytearray, memoryview, array.array, mmap.mmap] diff --git a/stdlib/3/gzip.pyi b/stdlib/3/gzip.pyi index 57d4e212c982..f2976dcb2ee4 100644 --- a/stdlib/3/gzip.pyi +++ b/stdlib/3/gzip.pyi @@ -2,7 +2,7 @@ import sys import zlib from typing import IO, Optional, TextIO, Union, overload import _compression -from _typeshed import AnyPath +from _typeshed import AnyPath, ReadableBuffer from typing_extensions import Literal _OpenBinaryMode = Literal["r", "rb", "a", "ab", "w", "wb", "x", "xb"] @@ -63,7 +63,7 @@ class GzipFile(_compression.BaseStream): @property def mtime(self) -> Optional[int]: ... crc: int - def write(self, data: bytes) -> int: ... + def write(self, data: ReadableBuffer) -> int: ... def read(self, size: Optional[int] = ...) -> bytes: ... def read1(self, size: int = ...) -> bytes: ... def peek(self, n: int) -> bytes: ... diff --git a/stdlib/3/io.pyi b/stdlib/3/io.pyi index 1ca15c9a3f9f..e1e732c9e677 100644 --- a/stdlib/3/io.pyi +++ b/stdlib/3/io.pyi @@ -6,8 +6,7 @@ import codecs import sys from mmap import mmap from types import TracebackType - -_bytearray_like = Union[bytearray, mmap] +from _typeshed import ReadableBuffer, WriteableBuffer DEFAULT_BUFFER_SIZE: int @@ -42,7 +41,7 @@ class IOBase: def tell(self) -> int: ... def truncate(self, __size: Optional[int] = ...) -> int: ... def writable(self) -> bool: ... - def writelines(self, __lines: Iterable[Union[bytes, bytearray]]) -> None: ... + def writelines(self, __lines: Iterable[ReadableBuffer]) -> None: ... def readline(self, __size: Optional[int] = ...) -> bytes: ... def __del__(self) -> None: ... @property @@ -51,16 +50,16 @@ class IOBase: class RawIOBase(IOBase): def readall(self) -> bytes: ... - def readinto(self, __buffer: bytearray) -> Optional[int]: ... - def write(self, __b: Union[bytes, bytearray]) -> Optional[int]: ... + def readinto(self, __buffer: WriteableBuffer) -> Optional[int]: ... + def write(self, __b: ReadableBuffer) -> Optional[int]: ... def read(self, __size: int = ...) -> Optional[bytes]: ... class BufferedIOBase(IOBase): raw: RawIOBase # This is not part of the BufferedIOBase API and may not exist on some implementations. def detach(self) -> RawIOBase: ... - def readinto(self, __buffer: _bytearray_like) -> int: ... - def write(self, __buffer: Union[bytes, bytearray]) -> int: ... - def readinto1(self, __buffer: _bytearray_like) -> int: ... + def readinto(self, __buffer: WriteableBuffer) -> int: ... + def write(self, __buffer: ReadableBuffer) -> int: ... + def readinto1(self, __buffer: WriteableBuffer) -> int: ... def read(self, __size: Optional[int] = ...) -> bytes: ... def read1(self, __size: int = ...) -> bytes: ... @@ -75,7 +74,7 @@ class FileIO(RawIOBase, BinaryIO): closefd: bool = ..., opener: Optional[Callable[[Union[int, str], str], int]] = ... ) -> None: ... - def write(self, __b: bytes) -> int: ... + def write(self, __b: ReadableBuffer) -> int: ... def read(self, __size: int = ...) -> bytes: ... def __enter__(self: _T) -> _T: ... @@ -105,7 +104,7 @@ class BufferedReader(BufferedIOBase, BinaryIO): class BufferedWriter(BufferedIOBase, BinaryIO): def __enter__(self: _T) -> _T: ... def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ... - def write(self, __buffer: Union[bytes, bytearray]) -> int: ... + def write(self, __buffer: ReadableBuffer) -> int: ... class BufferedRandom(BufferedReader, BufferedWriter): def __enter__(self: _T) -> _T: ... diff --git a/stdlib/3/lzma.pyi b/stdlib/3/lzma.pyi index 3a06895ec4fe..30e71e0ec8da 100644 --- a/stdlib/3/lzma.pyi +++ b/stdlib/3/lzma.pyi @@ -1,6 +1,6 @@ import io from typing import IO, Any, Mapping, Optional, Sequence, TextIO, TypeVar, Union, overload -from _typeshed import AnyPath +from _typeshed import AnyPath, ReadableBuffer from typing_extensions import Literal _OpenBinaryWritingMode = Literal["w", "wb", "x", "xb", "a", "ab"] @@ -88,7 +88,7 @@ class LZMAFile(io.BufferedIOBase, IO[bytes]): def read(self, size: Optional[int] = ...) -> bytes: ... def read1(self, size: int = ...) -> bytes: ... def readline(self, size: Optional[int] = ...) -> bytes: ... - def write(self, data: bytes) -> int: ... + def write(self, data: ReadableBuffer) -> int: ... def seek(self, offset: int, whence: int = ...) -> int: ... def tell(self) -> int: ...