Skip to content

Commit

Permalink
Introduce ReadableBuffer and WriteableBuffer Union aliases
Browse files Browse the repository at this point in the history
Since typing doesn't yet have a way to express buffer protocol objects
(python/typing#593), various interfaces have ended up with a mish-mash
of options: some list just bytes (or just bytearray, when writable),
some include mmap, some include memoryview, I think none of them include
array.array even though it's explicitly mentioned as bytes-like, etc. I
ran into problems because RawIOBase.readinto didn't allow for
memoryview.

To allow for some uniformity until the fundamental issue is resolved,
I've introduced _typeshed.ReadableBuffer and _typeshed.WriteableBuffer,
and applied them in stdlib/3/io.pyi as an example. If these get rolled
out in more places, it will mean that we have only one place where they
have to get tweaked in future, or swapped out for a public protocol.

This unfortunately does have the potential to break code that inherits
from RawIOBase/BufferedIOBase and overrides these methods, because the
base method is now more general and so the override now needs to accept
these types as well (which is why I've also updated gzip and lzma).
However, it should be a reasonably easy fix, and will make the
downstream annotations more correct.

I'm not 100% happy with the names: bytes-like is slightly stricter than
just buffer protocol (it must be able to export a C-contiguous buffer),
but in practice I'd be surprised if there are types for which there is a
difference at static analysis time (e.g. not every memoryview instance
is bytes-like, but that's a property of instances, not types).
  • Loading branch information
bmerry committed Jun 15, 2020
1 parent 51cf2f5 commit 0c0bcf4
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 14 deletions.
5 changes: 5 additions & 0 deletions stdlib/2and3/_typeshed/__init__.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
# If on Python versions < 3.10 and "from __future__ import annotations"
# is not used, types from this module must be quoted.

import array
import mmap
import sys
from typing import Protocol, Text, TypeVar, Union
from typing_extensions import Literal
Expand Down Expand Up @@ -67,3 +69,6 @@ class SupportsReadline(Protocol[_T_co]):
def readline(self, __length: int = ...) -> _T_co: ...
class SupportsWrite(Protocol[_T_contra]):
def write(self, __s: _T_contra) -> int: ...

ReadableBuffer = Union[bytes, bytearray, memoryview, array.array, mmap.mmap]
WriteableBuffer = Union[bytearray, memoryview, array.array, mmap.mmap]
4 changes: 2 additions & 2 deletions stdlib/3/gzip.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import sys
import zlib
from typing import IO, Optional, TextIO, Union, overload
import _compression
from _typeshed import AnyPath
from _typeshed import AnyPath, ReadableBuffer
from typing_extensions import Literal

_OpenBinaryMode = Literal["r", "rb", "a", "ab", "w", "wb", "x", "xb"]
Expand Down Expand Up @@ -63,7 +63,7 @@ class GzipFile(_compression.BaseStream):
@property
def mtime(self) -> Optional[int]: ...
crc: int
def write(self, data: bytes) -> int: ...
def write(self, data: ReadableBuffer) -> int: ...
def read(self, size: Optional[int] = ...) -> bytes: ...
def read1(self, size: int = ...) -> bytes: ...
def peek(self, n: int) -> bytes: ...
Expand Down
19 changes: 9 additions & 10 deletions stdlib/3/io.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ import codecs
import sys
from mmap import mmap
from types import TracebackType

_bytearray_like = Union[bytearray, mmap]
from _typeshed import ReadableBuffer, WriteableBuffer

DEFAULT_BUFFER_SIZE: int

Expand Down Expand Up @@ -42,7 +41,7 @@ class IOBase:
def tell(self) -> int: ...
def truncate(self, __size: Optional[int] = ...) -> int: ...
def writable(self) -> bool: ...
def writelines(self, __lines: Iterable[Union[bytes, bytearray]]) -> None: ...
def writelines(self, __lines: Iterable[ReadableBuffer]) -> None: ...
def readline(self, __size: Optional[int] = ...) -> bytes: ...
def __del__(self) -> None: ...
@property
Expand All @@ -51,16 +50,16 @@ class IOBase:

class RawIOBase(IOBase):
def readall(self) -> bytes: ...
def readinto(self, __buffer: bytearray) -> Optional[int]: ...
def write(self, __b: Union[bytes, bytearray]) -> Optional[int]: ...
def readinto(self, __buffer: WriteableBuffer) -> Optional[int]: ...
def write(self, __b: ReadableBuffer) -> Optional[int]: ...
def read(self, __size: int = ...) -> Optional[bytes]: ...

class BufferedIOBase(IOBase):
raw: RawIOBase # This is not part of the BufferedIOBase API and may not exist on some implementations.
def detach(self) -> RawIOBase: ...
def readinto(self, __buffer: _bytearray_like) -> int: ...
def write(self, __buffer: Union[bytes, bytearray]) -> int: ...
def readinto1(self, __buffer: _bytearray_like) -> int: ...
def readinto(self, __buffer: WriteableBuffer) -> int: ...
def write(self, __buffer: ReadableBuffer) -> int: ...
def readinto1(self, __buffer: WriteableBuffer) -> int: ...
def read(self, __size: Optional[int] = ...) -> bytes: ...
def read1(self, __size: int = ...) -> bytes: ...

Expand All @@ -75,7 +74,7 @@ class FileIO(RawIOBase, BinaryIO):
closefd: bool = ...,
opener: Optional[Callable[[Union[int, str], str], int]] = ...
) -> None: ...
def write(self, __b: bytes) -> int: ...
def write(self, __b: ReadableBuffer) -> int: ...
def read(self, __size: int = ...) -> bytes: ...
def __enter__(self: _T) -> _T: ...

Expand Down Expand Up @@ -105,7 +104,7 @@ class BufferedReader(BufferedIOBase, BinaryIO):
class BufferedWriter(BufferedIOBase, BinaryIO):
def __enter__(self: _T) -> _T: ...
def __init__(self, raw: RawIOBase, buffer_size: int = ...) -> None: ...
def write(self, __buffer: Union[bytes, bytearray]) -> int: ...
def write(self, __buffer: ReadableBuffer) -> int: ...

class BufferedRandom(BufferedReader, BufferedWriter):
def __enter__(self: _T) -> _T: ...
Expand Down
4 changes: 2 additions & 2 deletions stdlib/3/lzma.pyi
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import io
from typing import IO, Any, Mapping, Optional, Sequence, TextIO, TypeVar, Union, overload
from _typeshed import AnyPath
from _typeshed import AnyPath, ReadableBuffer
from typing_extensions import Literal

_OpenBinaryWritingMode = Literal["w", "wb", "x", "xb", "a", "ab"]
Expand Down Expand Up @@ -88,7 +88,7 @@ class LZMAFile(io.BufferedIOBase, IO[bytes]):
def read(self, size: Optional[int] = ...) -> bytes: ...
def read1(self, size: int = ...) -> bytes: ...
def readline(self, size: Optional[int] = ...) -> bytes: ...
def write(self, data: bytes) -> int: ...
def write(self, data: ReadableBuffer) -> int: ...
def seek(self, offset: int, whence: int = ...) -> int: ...
def tell(self) -> int: ...

Expand Down

0 comments on commit 0c0bcf4

Please sign in to comment.