diff --git a/README.md b/README.md index 3072daf79e..f8fe0f9b60 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,6 @@ The HTTPX project relies on these excellent libraries: * `rfc3986` - URL parsing & normalization. * `idna` - Internationalized domain name support. * `sniffio` - Async library autodetection. -* `urllib3` - Support for the `httpx.URLLib3Transport` class. *(Optional)* * `brotlipy` - Decoding for "brotli" compressed responses. *(Optional)* A huge amount of credit is due to `requests` for the API layout that diff --git a/docs/advanced.md b/docs/advanced.md index e43ecca6df..b2a07df371 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -809,6 +809,8 @@ HTTPX's `Client` also accepts a `transport` argument. This argument allows you to provide a custom Transport object that will be used to perform the actual sending of the requests. +### Usage + For some advanced configuration you might need to instantiate a transport class directly, and pass it to the client instance. The `httpcore` package provides a `local_address` configuration that is only available via this @@ -850,18 +852,19 @@ do not include any default values for configuring aspects such as the connection pooling details, so you'll need to provide more explicit configuration when using this API. -HTTPX also currently ships with a transport that uses the excellent -[`urllib3` library](https://urllib3.readthedocs.io/en/latest/), which can be -used with the sync `Client`... +### urllib3 transport + +This [public gist](https://gist.github.com/florimondmanca/d56764d78d748eb9f73165da388e546e) provides a transport that uses the excellent [`urllib3` library](https://urllib3.readthedocs.io/en/latest/), and can be used with the sync `Client`... ```pycon >>> import httpx ->>> client = httpx.Client(transport=httpx.URLLib3Transport()) +>>> from urllib3_transport import URLLib3Transport +>>> client = httpx.Client(transport=URLLib3Transport()) >>> client.get("https://example.org") ``` -Note that you'll need to install the `urllib3` package to use `URLLib3Transport`. +### Writing custom transports A transport instance must implement the Transport API defined by [`httpcore`](https://www.encode.io/httpcore/api/). You diff --git a/docs/compatibility.md b/docs/compatibility.md index dbb8941b8b..daef4994a7 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -83,3 +83,9 @@ Besides, `httpx.Request()` does not support the `auth`, `timeout`, `allow_redire ## Mocking If you need to mock HTTPX the same way that test utilities like `responses` and `requests-mock` does for `requests`, see [RESPX](https://github.com/lundberg/respx). + +## Networking layer + +`requests` defers most of its HTTP networking code to the excellent [`urllib3` library](https://urllib3.readthedocs.io/en/latest/). + +On the other hand, HTTPX uses [HTTPCore](https://github.com/encode/httpcore) as its core HTTP networking layer, which is a different project than `urllib3`. diff --git a/docs/index.md b/docs/index.md index 540c9bdc2d..ddea48c43f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -114,7 +114,6 @@ The HTTPX project relies on these excellent libraries: * `rfc3986` - URL parsing & normalization. * `idna` - Internationalized domain name support. * `sniffio` - Async library autodetection. -* `urllib3` - Support for the `httpx.URLLib3Transport` class. *(Optional)* * `brotlipy` - Decoding for "brotli" compressed responses. *(Optional)* A huge amount of credit is due to `requests` for the API layout that diff --git a/docs/third-party-packages.md b/docs/third-party-packages.md index cc6a79994e..61e6e860ab 100644 --- a/docs/third-party-packages.md +++ b/docs/third-party-packages.md @@ -23,3 +23,13 @@ An asynchronous GitHub API library. Includes [HTTPX support](https://gidgethub.r [GitHub](https://github.com/lundberg/respx) - [Documentation](https://lundberg.github.io/respx/) A utility for mocking out the Python HTTPX library. + +## Gists + + + +### urllib3-transport + +[GitHub](https://gist.github.com/florimondmanca/d56764d78d748eb9f73165da388e546e) + +This public gist provides an example implementation for a [custom transport](/advanced#custom-transports) implementation on top of the battle-tested [`urllib3`](https://urllib3.readthedocs.io) library. diff --git a/httpx/__init__.py b/httpx/__init__.py index 3802439466..bfd52806ef 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -38,7 +38,6 @@ from ._models import URL, Cookies, Headers, QueryParams, Request, Response from ._status_codes import StatusCode, codes from ._transports.asgi import ASGITransport -from ._transports.urllib3 import URLLib3ProxyTransport, URLLib3Transport from ._transports.wsgi import WSGITransport __all__ = [ @@ -101,8 +100,6 @@ "TransportError", "UnsupportedProtocol", "URL", - "URLLib3ProxyTransport", - "URLLib3Transport", "WriteError", "WriteTimeout", "WSGITransport", diff --git a/httpx/_client.py b/httpx/_client.py index eaa80fb3b3..b2fa2c8733 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -1,9 +1,11 @@ import functools import typing +import warnings from types import TracebackType import httpcore +from .__version__ import __version__ from ._auth import Auth, BasicAuth, FunctionAuth from ._config import ( DEFAULT_LIMITS, @@ -17,6 +19,7 @@ create_ssl_context, ) from ._content_streams import ContentStream +from ._decoders import SUPPORTED_DECODERS from ._exceptions import ( HTTPCORE_EXC_MAP, InvalidURL, @@ -54,6 +57,10 @@ logger = get_logger(__name__) KEEPALIVE_EXPIRY = 5.0 +USER_AGENT = f"python-httpx/{__version__}" +ACCEPT_ENCODING = ", ".join( + [key for key in SUPPORTED_DECODERS.keys() if key != "identity"] +) class BaseClient: @@ -73,12 +80,20 @@ def __init__( self._auth = self._build_auth(auth) self._params = QueryParams(params) - self._headers = Headers(headers) + self.headers = Headers(headers) self._cookies = Cookies(cookies) self._timeout = Timeout(timeout) self.max_redirects = max_redirects self._trust_env = trust_env self._netrc = NetRCInfo() + self._is_closed = True + + @property + def is_closed(self) -> bool: + """ + Check if the client being closed + """ + return self._is_closed @property def trust_env(self) -> bool: @@ -152,7 +167,16 @@ def headers(self) -> Headers: @headers.setter def headers(self, headers: HeaderTypes) -> None: - self._headers = Headers(headers) + client_headers = Headers( + { + b"Accept": b"*/*", + b"Accept-Encoding": ACCEPT_ENCODING.encode("ascii"), + b"Connection": b"keep-alive", + b"User-Agent": USER_AGENT.encode("ascii"), + } + ) + client_headers.update(headers) + self._headers = client_headers @property def cookies(self) -> Cookies: @@ -290,11 +314,9 @@ def _merge_headers( Merge a headers argument together with any headers on the client, to create the headers used for the outgoing request. """ - if headers or self.headers: - merged_headers = Headers(self.headers) - merged_headers.update(headers) - return merged_headers - return headers + merged_headers = Headers(self.headers) + merged_headers.update(headers) + return merged_headers def _merge_queryparams( self, params: QueryParamTypes = None @@ -696,6 +718,8 @@ def send( [0]: /advanced/#request-instances """ + self._is_closed = False + timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) auth = self._build_request_auth(request, auth) @@ -1026,16 +1050,20 @@ def close(self) -> None: """ Close transport and proxies. """ - self._transport.close() - for proxy in self._proxies.values(): - if proxy is not None: - proxy.close() + if not self.is_closed: + self._is_closed = True + + self._transport.close() + for proxy in self._proxies.values(): + if proxy is not None: + proxy.close() def __enter__(self) -> "Client": self._transport.__enter__() for proxy in self._proxies.values(): if proxy is not None: proxy.__enter__() + self._is_closed = False return self def __exit__( @@ -1044,10 +1072,16 @@ def __exit__( exc_value: BaseException = None, traceback: TracebackType = None, ) -> None: - self._transport.__exit__(exc_type, exc_value, traceback) - for proxy in self._proxies.values(): - if proxy is not None: - proxy.__exit__(exc_type, exc_value, traceback) + if not self.is_closed: + self._is_closed = True + + self._transport.__exit__(exc_type, exc_value, traceback) + for proxy in self._proxies.values(): + if proxy is not None: + proxy.__exit__(exc_type, exc_value, traceback) + + def __del__(self) -> None: + self.close() class AsyncClient(BaseClient): @@ -1302,6 +1336,8 @@ async def send( [0]: /advanced/#request-instances """ + self._is_closed = False + timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) auth = self._build_request_auth(request, auth) @@ -1634,16 +1670,20 @@ async def aclose(self) -> None: """ Close transport and proxies. """ - await self._transport.aclose() - for proxy in self._proxies.values(): - if proxy is not None: - await proxy.aclose() + if not self.is_closed: + self._is_closed = True + + await self._transport.aclose() + for proxy in self._proxies.values(): + if proxy is not None: + await proxy.aclose() async def __aenter__(self) -> "AsyncClient": await self._transport.__aenter__() for proxy in self._proxies.values(): if proxy is not None: await proxy.__aenter__() + self._is_closed = False return self async def __aexit__( @@ -1652,10 +1692,20 @@ async def __aexit__( exc_value: BaseException = None, traceback: TracebackType = None, ) -> None: - await self._transport.__aexit__(exc_type, exc_value, traceback) - for proxy in self._proxies.values(): - if proxy is not None: - await proxy.__aexit__(exc_type, exc_value, traceback) + if not self.is_closed: + self._is_closed = True + await self._transport.__aexit__(exc_type, exc_value, traceback) + for proxy in self._proxies.values(): + if proxy is not None: + await proxy.__aexit__(exc_type, exc_value, traceback) + + def __del__(self) -> None: + if not self.is_closed: + warnings.warn( + f"Unclosed {self!r}. " + "See https://www.python-httpx.org/async/#opening-and-closing-clients " + "for details." + ) class StreamContextManager: diff --git a/httpx/_exceptions.py b/httpx/_exceptions.py index 4d6837778a..260d14ee5f 100644 --- a/httpx/_exceptions.py +++ b/httpx/_exceptions.py @@ -55,7 +55,7 @@ class HTTPError(Exception): response = httpx.get("https://www.example.com") response.raise_for_status() except httpx.HTTPError as exc: - print(f"HTTP Exception for {exc.request.url} - {exc.message}") + print(f"HTTP Exception for {exc.request.url} - {exc}") ``` """ diff --git a/httpx/_models.py b/httpx/_models.py index 4a40263266..5b6a9b6571 100644 --- a/httpx/_models.py +++ b/httpx/_models.py @@ -14,7 +14,6 @@ import rfc3986 import rfc3986.exceptions -from .__version__ import __version__ from ._content_streams import ByteStream, ContentStream, encode from ._decoders import ( SUPPORTED_DECODERS, @@ -72,8 +71,12 @@ def __init__(self, url: URLTypes = "", params: QueryParamTypes = None) -> None: # We don't want to normalize relative URLs, since doing so # removes any leading `../` portion. self._uri_reference = self._uri_reference.normalize() - else: + elif isinstance(url, URL): self._uri_reference = url._uri_reference + else: + raise TypeError( + f"Invalid type for url. Expected str or httpx.URL, got {type(url)}" + ) # Add any query parameters, merging with any in the URL if needed. if params: @@ -103,13 +106,11 @@ def userinfo(self) -> str: @property def username(self) -> str: - userinfo = self._uri_reference.userinfo or "" - return unquote(userinfo.partition(":")[0]) + return unquote(self.userinfo.partition(":")[0]) @property def password(self) -> str: - userinfo = self._uri_reference.userinfo or "" - return unquote(userinfo.partition(":")[2]) + return unquote(self.userinfo.partition(":")[2]) @property def host(self) -> str: @@ -580,12 +581,6 @@ def getlist(self, key: str, split_commas: bool = False) -> typing.List[str]: return self.get_list(key, split_commas=split_commas) -USER_AGENT = f"python-httpx/{__version__}" -ACCEPT_ENCODING = ", ".join( - [key for key in SUPPORTED_DECODERS.keys() if key != "identity"] -) - - class Request: def __init__( self, @@ -627,26 +622,12 @@ def prepare(self) -> None: has_content_length = ( "content-length" in self.headers or "transfer-encoding" in self.headers ) - has_user_agent = "user-agent" in self.headers - has_accept = "accept" in self.headers - has_accept_encoding = "accept-encoding" in self.headers - has_connection = "connection" in self.headers - - if not has_host: - url = self.url - if url.userinfo: - url = url.copy_with(username=None, password=None) - auto_headers.append((b"host", url.authority.encode("ascii"))) + + if not has_host and self.url.authority: + host = self.url.copy_with(username=None, password=None).authority + auto_headers.append((b"host", host.encode("ascii"))) if not has_content_length and self.method in ("POST", "PUT", "PATCH"): auto_headers.append((b"content-length", b"0")) - if not has_user_agent: - auto_headers.append((b"user-agent", USER_AGENT.encode("ascii"))) - if not has_accept: - auto_headers.append((b"accept", b"*/*")) - if not has_accept_encoding: - auto_headers.append((b"accept-encoding", ACCEPT_ENCODING.encode())) - if not has_connection: - auto_headers.append((b"connection", b"keep-alive")) self.headers = Headers(auto_headers + self.headers.raw) diff --git a/httpx/_transports/urllib3.py b/httpx/_transports/urllib3.py deleted file mode 100644 index c5b7af6cc2..0000000000 --- a/httpx/_transports/urllib3.py +++ /dev/null @@ -1,149 +0,0 @@ -import socket -from typing import Iterator, List, Mapping, Optional, Tuple - -import httpcore - -from .._config import create_ssl_context -from .._content_streams import ByteStream, IteratorStream -from .._exceptions import NetworkError, map_exceptions -from .._types import CertTypes, VerifyTypes - -try: - import urllib3 - from urllib3.exceptions import MaxRetryError, SSLError -except ImportError: # pragma: nocover - urllib3 = None - - -class URLLib3Transport(httpcore.SyncHTTPTransport): - def __init__( - self, - *, - verify: VerifyTypes = True, - cert: CertTypes = None, - trust_env: bool = None, - pool_connections: int = 10, - pool_maxsize: int = 10, - pool_block: bool = False, - ): - assert ( - urllib3 is not None - ), "urllib3 must be installed in order to use URLLib3Transport" - - self.pool = urllib3.PoolManager( - ssl_context=create_ssl_context( - verify=verify, cert=cert, trust_env=trust_env, http2=False - ), - num_pools=pool_connections, - maxsize=pool_maxsize, - block=pool_block, - ) - - def request( - self, - method: bytes, - url: Tuple[bytes, bytes, Optional[int], bytes], - headers: List[Tuple[bytes, bytes]] = None, - stream: httpcore.SyncByteStream = None, - timeout: Mapping[str, Optional[float]] = None, - ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], httpcore.SyncByteStream]: - headers = [] if headers is None else headers - stream = ByteStream(b"") if stream is None else stream - timeout = {} if timeout is None else timeout - - urllib3_timeout = urllib3.util.Timeout( - connect=timeout.get("connect"), read=timeout.get("read") - ) - - chunked = False - content_length = 0 - for header_key, header_value in headers: - header_key = header_key.lower() - if header_key == b"transfer-encoding": - chunked = header_value == b"chunked" - if header_key == b"content-length": - content_length = int(header_value.decode("ascii")) - body = stream if chunked or content_length else None - - scheme, host, port, path = url - default_port = {b"http": 80, "https": 443}.get(scheme) - if port is None or port == default_port: - url_str = "%s://%s%s" % ( - scheme.decode("ascii"), - host.decode("ascii"), - path.decode("ascii"), - ) - else: - url_str = "%s://%s:%d%s" % ( - scheme.decode("ascii"), - host.decode("ascii"), - port, - path.decode("ascii"), - ) - - with map_exceptions( - { - MaxRetryError: NetworkError, - SSLError: NetworkError, - socket.error: NetworkError, - } - ): - conn = self.pool.urlopen( - method=method.decode(), - url=url_str, - headers={ - key.decode("ascii"): value.decode("ascii") for key, value in headers - }, - body=body, - redirect=False, - assert_same_host=False, - retries=0, - preload_content=False, - chunked=chunked, - timeout=urllib3_timeout, - pool_timeout=timeout.get("pool"), - ) - - def response_bytes() -> Iterator[bytes]: - with map_exceptions({socket.error: NetworkError}): - for chunk in conn.stream(4096, decode_content=False): - yield chunk - - status_code = conn.status - headers = list(conn.headers.items()) - response_stream = IteratorStream( - iterator=response_bytes(), close_func=conn.release_conn - ) - return (b"HTTP/1.1", status_code, conn.reason, headers, response_stream) - - def close(self) -> None: - self.pool.clear() - - -class URLLib3ProxyTransport(URLLib3Transport): - def __init__( - self, - *, - proxy_url: str, - proxy_headers: dict = None, - verify: VerifyTypes = True, - cert: CertTypes = None, - trust_env: bool = None, - pool_connections: int = 10, - pool_maxsize: int = 10, - pool_block: bool = False, - ): - assert ( - urllib3 is not None - ), "urllib3 must be installed in order to use URLLib3ProxyTransport" - - self.pool = urllib3.ProxyManager( - proxy_url=proxy_url, - proxy_headers=proxy_headers, - ssl_context=create_ssl_context( - verify=verify, cert=cert, trust_env=trust_env, http2=False - ), - num_pools=pool_connections, - maxsize=pool_maxsize, - block=pool_block, - ) diff --git a/scripts/coverage b/scripts/coverage index a476e8b278..25a2691074 100755 --- a/scripts/coverage +++ b/scripts/coverage @@ -8,4 +8,4 @@ export SOURCE_FILES="httpx tests" set -x -${PREFIX}coverage report --omit=httpx/_transports/urllib3.py --show-missing --skip-covered --fail-under=100 +${PREFIX}coverage report --show-missing --skip-covered --fail-under=100 diff --git a/tests/client/test_async_client.py b/tests/client/test_async_client.py index 126c50e95b..4272301f6f 100644 --- a/tests/client/test_async_client.py +++ b/tests/client/test_async_client.py @@ -206,3 +206,45 @@ async def __aexit__(self, *args): "transport.aclose", "transport.__aexit__", ] + + +@pytest.mark.usefixtures("async_environment") +async def test_that_async_client_is_closed_by_default(): + client = httpx.AsyncClient() + + assert client.is_closed + + +@pytest.mark.usefixtures("async_environment") +async def test_that_send_cause_async_client_to_be_not_closed(): + client = httpx.AsyncClient() + + await client.get("http://example.com") + + assert not client.is_closed + + await client.aclose() + + +@pytest.mark.usefixtures("async_environment") +async def test_that_async_client_is_not_closed_in_with_block(): + async with httpx.AsyncClient() as client: + assert not client.is_closed + + +@pytest.mark.usefixtures("async_environment") +async def test_that_async_client_is_closed_after_with_block(): + async with httpx.AsyncClient() as client: + pass + + assert client.is_closed + + +@pytest.mark.usefixtures("async_environment") +async def test_that_async_client_caused_warning_when_being_deleted(): + async_client = httpx.AsyncClient() + + await async_client.get("http://example.com") + + with pytest.warns(UserWarning): + del async_client diff --git a/tests/client/test_client.py b/tests/client/test_client.py index fe427ec664..d5a8c0e004 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -247,3 +247,29 @@ def __exit__(self, *args): "transport.close", "transport.__exit__", ] + + +def test_that_client_is_closed_by_default(): + client = httpx.Client() + + assert client.is_closed + + +def test_that_send_cause_client_to_be_not_closed(): + client = httpx.Client() + + client.get("http://example.com") + + assert not client.is_closed + + +def test_that_client_is_not_closed_in_with_block(): + with httpx.Client() as client: + assert not client.is_closed + + +def test_that_client_is_closed_after_with_block(): + with httpx.Client() as client: + pass + + assert client.is_closed diff --git a/tests/client/test_headers.py b/tests/client/test_headers.py index 34a17c3916..c86eae33c1 100755 --- a/tests/client/test_headers.py +++ b/tests/client/test_headers.py @@ -124,6 +124,28 @@ def test_header_update(): } +def test_remove_default_header(): + """ + Remove a default header from the Client. + """ + url = "http://example.org/echo_headers" + + client = httpx.Client(transport=MockTransport()) + del client.headers["User-Agent"] + + response = client.get(url) + + assert response.status_code == 200 + assert response.json() == { + "headers": { + "accept": "*/*", + "accept-encoding": "gzip, deflate, br", + "connection": "keep-alive", + "host": "example.org", + } + } + + def test_header_does_not_exist(): headers = httpx.Headers({"foo": "bar"}) with pytest.raises(KeyError): diff --git a/tests/client/test_proxies.py b/tests/client/test_proxies.py index c4a64e1a26..6d30438362 100644 --- a/tests/client/test_proxies.py +++ b/tests/client/test_proxies.py @@ -123,14 +123,16 @@ def test_transport_for_request(url, proxies, expected): @pytest.mark.asyncio async def test_async_proxy_close(): try: - client = httpx.AsyncClient(proxies={"all://": PROXY_URL}) + client = httpx.AsyncClient(proxies={"https://": PROXY_URL}) + await client.get("http://example.com") finally: await client.aclose() def test_sync_proxy_close(): try: - client = httpx.Client(proxies={"all://": PROXY_URL}) + client = httpx.Client(proxies={"https://": PROXY_URL}) + client.get("http://example.com") finally: client.close() diff --git a/tests/models/test_requests.py b/tests/models/test_requests.py index f2ee98edb6..8b8325e314 100644 --- a/tests/models/test_requests.py +++ b/tests/models/test_requests.py @@ -34,6 +34,18 @@ def test_json_encoded_data(): assert request.content == b'{"test": 123}' +def test_headers(): + request = httpx.Request("POST", "http://example.org", json={"test": 123}) + + assert request.headers == httpx.Headers( + { + "Host": "example.org", + "Content-Type": "application/json", + "Content-Length": "13", + } + ) + + def test_read_and_stream_data(): # Ensure a request may still be streamed if it has been read. # Needed for cases such as authentication classes that read the request body. diff --git a/tests/models/test_url.py b/tests/models/test_url.py index fa75e556a6..8d34a75a79 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -204,3 +204,11 @@ def test_url_copywith_for_userinfo(): def test_url_invalid(): with pytest.raises(httpx.InvalidURL): httpx.URL("https://😇/") + + +def test_url_invalid_type(): + class ExternalURLClass: # representing external URL class + pass + + with pytest.raises(TypeError): + httpx.URL(ExternalURLClass()) # type: ignore