Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to httpcore 0.10 #1126

Merged
merged 9 commits into from
Aug 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions httpx/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@
HTTPError,
HTTPStatusError,
InvalidURL,
LocalProtocolError,
NetworkError,
NotRedirectResponse,
PoolTimeout,
ProtocolError,
ProxyError,
ReadError,
ReadTimeout,
RemoteProtocolError,
RequestBodyUnavailable,
RequestError,
RequestNotRead,
Expand All @@ -29,6 +31,7 @@
TimeoutException,
TooManyRedirects,
TransportError,
UnsupportedProtocol,
WriteError,
WriteTimeout,
)
Expand Down Expand Up @@ -72,6 +75,9 @@
"HTTPError",
"HTTPStatusError",
"InvalidURL",
"UnsupportedProtocol",
"LocalProtocolError",
"RemoteProtocolError",
"NetworkError",
"NotRedirectResponse",
"PoolTimeout",
Expand Down
33 changes: 8 additions & 25 deletions httpx/_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from ._content_streams import ContentStream
from ._exceptions import (
HTTPCORE_EXC_MAP,
InvalidURL,
RequestBodyUnavailable,
TooManyRedirects,
map_exceptions,
Expand All @@ -44,7 +43,6 @@
from ._utils import (
NetRCInfo,
URLPattern,
enforce_http_url,
get_environment_proxies,
get_logger,
same_origin,
Expand Down Expand Up @@ -344,11 +342,6 @@ def _redirect_url(self, request: Request, response: Response) -> URL:

url = URL(location)

# Check that we can handle the scheme
if url.scheme and url.scheme not in ("http", "https"):
message = f'Scheme "{url.scheme}" not supported.'
raise InvalidURL(message, request=request)

# Handle malformed 'Location' headers that are "absolute" form, have no host.
# See: https://github.com/encode/httpx/issues/771
if url.scheme and not url.host:
Expand Down Expand Up @@ -540,8 +533,8 @@ def _init_transport(

return httpcore.SyncConnectionPool(
ssl_context=ssl_context,
max_keepalive=limits.max_keepalive,
max_connections=limits.max_connections,
max_keepalive_connections=limits.max_keepalive_connections,
keepalive_expiry=KEEPALIVE_EXPIRY,
http2=http2,
)
Expand All @@ -562,20 +555,17 @@ def _init_proxy_transport(
proxy_headers=proxy.headers.raw,
proxy_mode=proxy.mode,
ssl_context=ssl_context,
max_keepalive=limits.max_keepalive,
max_connections=limits.max_connections,
max_keepalive_connections=limits.max_keepalive_connections,
keepalive_expiry=KEEPALIVE_EXPIRY,
http2=http2,
)

def _transport_for_url(self, request: Request) -> httpcore.SyncHTTPTransport:
def _transport_for_url(self, url: URL) -> httpcore.SyncHTTPTransport:
"""
Returns the transport instance that should be used for a given URL.
This will either be the standard connection pool, or a proxy.
"""
url = request.url
enforce_http_url(request)

for pattern, transport in self._proxies.items():
if pattern.matches(url):
return self._transport if transport is None else transport
Expand Down Expand Up @@ -620,10 +610,6 @@ def send(
allow_redirects: bool = True,
timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET,
) -> Response:
if request.url.scheme not in ("http", "https"):
message = 'URL scheme must be "http" or "https".'
raise InvalidURL(message, request=request)

timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout)

auth = self._build_auth(request, auth)
Expand Down Expand Up @@ -714,7 +700,7 @@ def _send_single_request(self, request: Request, timeout: Timeout) -> Response:
"""
Sends a single request, without handling any redirections.
"""
transport = self._transport_for_url(request)
transport = self._transport_for_url(request.url)

with map_exceptions(HTTPCORE_EXC_MAP, request=request):
(
Expand Down Expand Up @@ -1072,8 +1058,8 @@ def _init_transport(

return httpcore.AsyncConnectionPool(
ssl_context=ssl_context,
max_keepalive=limits.max_keepalive,
max_connections=limits.max_connections,
max_keepalive_connections=limits.max_keepalive_connections,
keepalive_expiry=KEEPALIVE_EXPIRY,
http2=http2,
)
Expand All @@ -1094,20 +1080,17 @@ def _init_proxy_transport(
proxy_headers=proxy.headers.raw,
proxy_mode=proxy.mode,
ssl_context=ssl_context,
max_keepalive=limits.max_keepalive,
max_connections=limits.max_connections,
max_keepalive_connections=limits.max_keepalive_connections,
keepalive_expiry=KEEPALIVE_EXPIRY,
http2=http2,
)

def _transport_for_url(self, request: Request) -> httpcore.AsyncHTTPTransport:
def _transport_for_url(self, url: URL) -> httpcore.AsyncHTTPTransport:
"""
Returns the transport instance that should be used for a given URL.
This will either be the standard connection pool, or a proxy.
"""
url = request.url
enforce_http_url(request)

for pattern, transport in self._proxies.items():
if pattern.matches(url):
return self._transport if transport is None else transport
Expand Down Expand Up @@ -1245,7 +1228,7 @@ async def _send_single_request(
"""
Sends a single request, without handling any redirections.
"""
transport = self._transport_for_url(request)
transport = self._transport_for_url(request.url)

with map_exceptions(HTTPCORE_EXC_MAP, request=request):
(
Expand Down
35 changes: 23 additions & 12 deletions httpx/_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -323,42 +323,53 @@ class Limits:

**Parameters:**

* **max_keepalive** - Allow the connection pool to maintain keep-alive connections
below this point.
* **max_connections** - The maximum number of concurrent connections that may be
established.
established.
* **max_keepalive_connections** - Allow the connection pool to maintain
keep-alive connections below this point. Should be less than or equal
to `max_connections`.
"""

def __init__(
self, *, max_keepalive: int = None, max_connections: int = None,
self,
*,
max_connections: int = None,
max_keepalive_connections: int = None,
# Deprecated parameter naming, in favour of more explicit version:
max_keepalive: int = None,
):
self.max_keepalive = max_keepalive
if max_keepalive is not None:
warnings.warn(
"'max_keepalive' is deprecated. Use 'max_keepalive_connections'.",
DeprecationWarning,
)
max_keepalive_connections = max_keepalive

self.max_connections = max_connections
self.max_keepalive_connections = max_keepalive_connections

def __eq__(self, other: typing.Any) -> bool:
return (
isinstance(other, self.__class__)
and self.max_keepalive == other.max_keepalive
and self.max_connections == other.max_connections
and self.max_keepalive_connections == other.max_keepalive_connections
)

def __repr__(self) -> str:
class_name = self.__class__.__name__
return (
f"{class_name}(max_keepalive={self.max_keepalive}, "
f"max_connections={self.max_connections})"
f"{class_name}(max_connections={self.max_connections}, "
f"max_keepalive_connections={self.max_keepalive_connections})"
)


class PoolLimits(Limits):
def __init__(
self, *, max_keepalive: int = None, max_connections: int = None,
) -> None:
def __init__(self, **kwargs: typing.Any) -> None:
warn_deprecated(
"httpx.PoolLimits(...) is deprecated and will raise errors in the future. "
"Use httpx.Limits(...) instead."
)
super().__init__(max_keepalive=max_keepalive, max_connections=max_connections)
super().__init__(**kwargs)


class Proxy:
Expand Down
49 changes: 41 additions & 8 deletions httpx/_exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
· WriteError
· CloseError
- ProtocolError
· LocalProtocolError
· RemoteProtocolError
- ProxyError
- UnsupportedProtocol
+ DecodingError
+ TooManyRedirects
+ RequestBodyUnavailable
+ InvalidURL
x HTTPStatusError
* NotRedirectResponse
* CookieConflict
Expand Down Expand Up @@ -153,9 +155,35 @@ class ProxyError(TransportError):
"""


class UnsupportedProtocol(TransportError):
"""
Attempted to make a request to an unsupported protocol.

For example issuing a request to `ftp://www.example.com`.
"""


class ProtocolError(TransportError):
"""
A protocol was violated by the server.
The protocol was violated.
"""


class LocalProtocolError(ProtocolError):
"""
A protocol was violated by the client.

For example if the user instantiated a `Request` instance explicitly,
failed to include the mandatory `Host:` header, and then issued it directly
using `client.send()`.
"""


class RemoteProtocolError(ProtocolError):
"""
The protocol was violated by the server.

For exaample, returning malformed HTTP.
"""


Expand All @@ -181,12 +209,6 @@ class RequestBodyUnavailable(RequestError):
"""


class InvalidURL(RequestError):
"""
URL was missing a hostname, or was not one of HTTP/HTTPS.
"""


# Client errors


Expand Down Expand Up @@ -297,6 +319,14 @@ def __init__(self) -> None:
super().__init__(message)


# The `InvalidURL` class is no longer required. It was being used to enforce only
# 'http'/'https' URLs being requested, but is now treated instead at the
# transport layer using `UnsupportedProtocol()`.`

# We are currently still exposing this class, but it will be removed in 1.0.
InvalidURL = UnsupportedProtocol


@contextlib.contextmanager
def map_exceptions(
mapping: typing.Mapping[typing.Type[Exception], typing.Type[Exception]],
Expand Down Expand Up @@ -335,5 +365,8 @@ def map_exceptions(
httpcore.WriteError: WriteError,
httpcore.CloseError: CloseError,
httpcore.ProxyError: ProxyError,
httpcore.UnsupportedProtocol: UnsupportedProtocol,
httpcore.ProtocolError: ProtocolError,
httpcore.LocalProtocolError: LocalProtocolError,
httpcore.RemoteProtocolError: RemoteProtocolError,
}
27 changes: 4 additions & 23 deletions httpx/_transports/asgi.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,4 @@
from typing import (
TYPE_CHECKING,
AsyncIterator,
Callable,
Dict,
List,
Optional,
Tuple,
Union,
)
from typing import TYPE_CHECKING, Callable, List, Mapping, Optional, Tuple, Union

import httpcore
import sniffio
Expand All @@ -31,10 +22,6 @@ def create_event() -> "Event":
return asyncio.Event()


async def async_byte_iterator(bytestring: bytes) -> AsyncIterator[bytes]:
yield bytestring


class ASGITransport(httpcore.AsyncHTTPTransport):
"""
A custom AsyncTransport that handles sending requests directly to an ASGI app.
Expand Down Expand Up @@ -86,14 +73,10 @@ async def request(
url: Tuple[bytes, bytes, Optional[int], bytes],
headers: List[Tuple[bytes, bytes]] = None,
stream: httpcore.AsyncByteStream = None,
timeout: Dict[str, Optional[float]] = None,
timeout: Mapping[str, Optional[float]] = None,
) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], httpcore.AsyncByteStream]:
headers = [] if headers is None else headers
stream = (
httpcore.AsyncByteStream(async_byte_iterator(b""))
if stream is None
else stream
)
stream = httpcore.PlainByteStream(content=b"") if stream is None else stream

# ASGI scope.
scheme, host, port, full_path = url
Expand Down Expand Up @@ -170,8 +153,6 @@ async def send(message: dict) -> None:
assert status_code is not None
assert response_headers is not None

response_body = b"".join(body_parts)

stream = httpcore.AsyncByteStream(async_byte_iterator(response_body))
stream = httpcore.PlainByteStream(content=b"".join(body_parts))

return (b"HTTP/1.1", status_code, b"", response_headers, stream)
4 changes: 2 additions & 2 deletions httpx/_transports/urllib3.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import socket
from typing import Dict, Iterator, List, Optional, Tuple
from typing import Iterator, List, Mapping, Optional, Tuple

import httpcore

Expand Down Expand Up @@ -45,7 +45,7 @@ def request(
url: Tuple[bytes, bytes, Optional[int], bytes],
headers: List[Tuple[bytes, bytes]] = None,
stream: httpcore.SyncByteStream = None,
timeout: Dict[str, Optional[float]] = None,
timeout: Mapping[str, Optional[float]] = None,
) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], httpcore.SyncByteStream]:
headers = [] if headers is None else headers
stream = ByteStream(b"") if stream is None else stream
Expand Down
Loading