From 5fc59042e64793edd784b48c5001a8c0c30f8d18 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 20 May 2020 16:37:18 +0100 Subject: [PATCH 1/2] Port becomes optional --- httpcore/_async/base.py | 2 +- httpcore/_async/connection.py | 4 ++-- httpcore/_async/connection_pool.py | 6 +++--- httpcore/_async/http2.py | 5 ++++- httpcore/_async/http_proxy.py | 20 ++++++++++++++------ httpcore/_sync/base.py | 2 +- httpcore/_sync/connection.py | 4 ++-- httpcore/_sync/connection_pool.py | 6 +++--- httpcore/_sync/http2.py | 5 ++++- httpcore/_sync/http_proxy.py | 20 ++++++++++++++------ httpcore/_types.py | 2 +- httpcore/_utils.py | 8 ++++++++ 12 files changed, 57 insertions(+), 27 deletions(-) diff --git a/httpcore/_async/base.py b/httpcore/_async/base.py index 3b324538..4e7c7bef 100644 --- a/httpcore/_async/base.py +++ b/httpcore/_async/base.py @@ -87,7 +87,7 @@ async def request( **Parameters:** * **method** - `bytes` - The HTTP method, such as `b'GET'`. - * **url** - `Tuple[bytes, bytes, int, bytes]` - The URL as a 4-tuple of + * **url** - `Tuple[bytes, bytes, Optional[int], bytes]` - The URL as a 4-tuple of (scheme, host, port, path). * **headers** - `Optional[List[Tuple[bytes, bytes]]]` - Any HTTP headers to send with the request. diff --git a/httpcore/_async/connection.py b/httpcore/_async/connection.py index d448620f..aab0f076 100644 --- a/httpcore/_async/connection.py +++ b/httpcore/_async/connection.py @@ -3,7 +3,7 @@ from .._backends.auto import AsyncLock, AsyncSocketStream, AutoBackend from .._types import URL, Headers, Origin, TimeoutDict -from .._utils import get_logger +from .._utils import get_logger, url_to_origin from .base import ( AsyncByteStream, AsyncHTTPTransport, @@ -55,7 +55,7 @@ async def request( stream: AsyncByteStream = None, timeout: TimeoutDict = None, ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], AsyncByteStream]: - assert url[:3] == self.origin + assert url_to_origin(url) == self.origin async with self.request_lock: if self.state == ConnectionState.PENDING: if not self.socket: diff --git a/httpcore/_async/connection_pool.py b/httpcore/_async/connection_pool.py index 1b9bfffa..62959d64 100644 --- a/httpcore/_async/connection_pool.py +++ b/httpcore/_async/connection_pool.py @@ -5,7 +5,7 @@ from .._exceptions import PoolTimeout from .._threadlock import ThreadLock from .._types import URL, Headers, Origin, TimeoutDict -from .._utils import get_logger +from .._utils import get_logger, url_to_origin from .base import ( AsyncByteStream, AsyncHTTPTransport, @@ -124,8 +124,8 @@ async def request( stream: AsyncByteStream = None, timeout: TimeoutDict = None, ) -> Tuple[bytes, int, bytes, Headers, AsyncByteStream]: - timeout = {} if timeout is None else timeout - origin = url[:3] + assert url[0] in (b'http', b'https') + origin = url_to_origin(url) if self._keepalive_expiry is not None: await self._keepalive_sweep() diff --git a/httpcore/_async/http2.py b/httpcore/_async/http2.py index c3c5277c..fa50ab6c 100644 --- a/httpcore/_async/http2.py +++ b/httpcore/_async/http2.py @@ -307,7 +307,10 @@ async def send_headers( ) -> None: scheme, hostname, port, path = url default_port = {b"http": 80, b"https": 443}.get(scheme) - authority = b"%s:%d" % (hostname, port) if port != default_port else hostname + if port is None or port == default_port: + authority = hostname + else: + authority = b"%s:%d" % (hostname, port) headers = [ (b":method", method), diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index 898681b2..1bb93853 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -3,7 +3,7 @@ from .._exceptions import ProxyError from .._types import URL, Headers, Origin, TimeoutDict -from .._utils import get_logger +from .._utils import get_logger, url_to_origin from .base import AsyncByteStream from .connection import AsyncHTTPConnection from .connection_pool import AsyncConnectionPool, ResponseByteStream @@ -51,7 +51,7 @@ class AsyncHTTPProxy(AsyncConnectionPool): def __init__( self, - proxy_origin: Origin, + proxy_url: URL, proxy_headers: Headers = None, proxy_mode: str = "DEFAULT", ssl_context: SSLContext = None, @@ -62,7 +62,7 @@ def __init__( ): assert proxy_mode in ("DEFAULT", "FORWARD_ONLY", "TUNNEL_ONLY") - self.proxy_origin = proxy_origin + self.proxy_origin = url_to_origin(proxy_url) self.proxy_headers = [] if proxy_headers is None else proxy_headers self.proxy_mode = proxy_mode super().__init__( @@ -137,7 +137,12 @@ async def _forward_request( # GET https://www.example.org/path HTTP/1.1 # [proxy headers] # [headers] - target = b"%b://%b:%d%b" % url + scheme, host, port, path = url + if port is None: + target = b"%b://%b%b" % (scheme, host, path) + else: + target = b"%b://%b:%d%b" % (scheme, host, port, path) + url = self.proxy_origin + (target,) headers = merge_headers(self.proxy_headers, headers) @@ -161,7 +166,7 @@ async def _tunnel_request( Tunnelled proxy requests require an initial CONNECT request to establish the connection, and then send regular requests. """ - origin = url[:3] + origin = url_to_origin(url) connection = await self._get_connection_from_pool(origin) if connection is None: @@ -176,7 +181,10 @@ async def _tunnel_request( # CONNECT www.example.org:80 HTTP/1.1 # [proxy-headers] - target = b"%b:%d" % (url[1], url[2]) + if url[2] is None: + target = url[1] + else: + target = b"%b:%d" % (url[1], url[2]) connect_url = self.proxy_origin + (target,) connect_headers = [(b"Host", target), (b"Accept", b"*/*")] connect_headers = merge_headers(connect_headers, self.proxy_headers) diff --git a/httpcore/_sync/base.py b/httpcore/_sync/base.py index bb24652c..d0834354 100644 --- a/httpcore/_sync/base.py +++ b/httpcore/_sync/base.py @@ -87,7 +87,7 @@ def request( **Parameters:** * **method** - `bytes` - The HTTP method, such as `b'GET'`. - * **url** - `Tuple[bytes, bytes, int, bytes]` - The URL as a 4-tuple of + * **url** - `Tuple[bytes, bytes, Optional[int], bytes]` - The URL as a 4-tuple of (scheme, host, port, path). * **headers** - `Optional[List[Tuple[bytes, bytes]]]` - Any HTTP headers to send with the request. diff --git a/httpcore/_sync/connection.py b/httpcore/_sync/connection.py index 4893d614..aeb5f9d2 100644 --- a/httpcore/_sync/connection.py +++ b/httpcore/_sync/connection.py @@ -3,7 +3,7 @@ from .._backends.auto import SyncLock, SyncSocketStream, SyncBackend from .._types import URL, Headers, Origin, TimeoutDict -from .._utils import get_logger +from .._utils import get_logger, url_to_origin from .base import ( SyncByteStream, SyncHTTPTransport, @@ -55,7 +55,7 @@ def request( stream: SyncByteStream = None, timeout: TimeoutDict = None, ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], SyncByteStream]: - assert url[:3] == self.origin + assert url_to_origin(url) == self.origin with self.request_lock: if self.state == ConnectionState.PENDING: if not self.socket: diff --git a/httpcore/_sync/connection_pool.py b/httpcore/_sync/connection_pool.py index 46cad6fc..212570c1 100644 --- a/httpcore/_sync/connection_pool.py +++ b/httpcore/_sync/connection_pool.py @@ -5,7 +5,7 @@ from .._exceptions import PoolTimeout from .._threadlock import ThreadLock from .._types import URL, Headers, Origin, TimeoutDict -from .._utils import get_logger +from .._utils import get_logger, url_to_origin from .base import ( SyncByteStream, SyncHTTPTransport, @@ -124,8 +124,8 @@ def request( stream: SyncByteStream = None, timeout: TimeoutDict = None, ) -> Tuple[bytes, int, bytes, Headers, SyncByteStream]: - timeout = {} if timeout is None else timeout - origin = url[:3] + assert url[0] in (b'http', b'https') + origin = url_to_origin(url) if self._keepalive_expiry is not None: self._keepalive_sweep() diff --git a/httpcore/_sync/http2.py b/httpcore/_sync/http2.py index e12c92bf..0acaff95 100644 --- a/httpcore/_sync/http2.py +++ b/httpcore/_sync/http2.py @@ -307,7 +307,10 @@ def send_headers( ) -> None: scheme, hostname, port, path = url default_port = {b"http": 80, b"https": 443}.get(scheme) - authority = b"%s:%d" % (hostname, port) if port != default_port else hostname + if port is None or port == default_port: + authority = hostname + else: + authority = b"%s:%d" % (hostname, port) headers = [ (b":method", method), diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index ab938397..71118510 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -3,7 +3,7 @@ from .._exceptions import ProxyError from .._types import URL, Headers, Origin, TimeoutDict -from .._utils import get_logger +from .._utils import get_logger, url_to_origin from .base import SyncByteStream from .connection import SyncHTTPConnection from .connection_pool import SyncConnectionPool, ResponseByteStream @@ -51,7 +51,7 @@ class SyncHTTPProxy(SyncConnectionPool): def __init__( self, - proxy_origin: Origin, + proxy_url: URL, proxy_headers: Headers = None, proxy_mode: str = "DEFAULT", ssl_context: SSLContext = None, @@ -62,7 +62,7 @@ def __init__( ): assert proxy_mode in ("DEFAULT", "FORWARD_ONLY", "TUNNEL_ONLY") - self.proxy_origin = proxy_origin + self.proxy_origin = url_to_origin(proxy_url) self.proxy_headers = [] if proxy_headers is None else proxy_headers self.proxy_mode = proxy_mode super().__init__( @@ -137,7 +137,12 @@ def _forward_request( # GET https://www.example.org/path HTTP/1.1 # [proxy headers] # [headers] - target = b"%b://%b:%d%b" % url + scheme, host, port, path = url + if port is None: + target = b"%b://%b%b" % (scheme, host, path) + else: + target = b"%b://%b:%d%b" % (scheme, host, port, path) + url = self.proxy_origin + (target,) headers = merge_headers(self.proxy_headers, headers) @@ -161,7 +166,7 @@ def _tunnel_request( Tunnelled proxy requests require an initial CONNECT request to establish the connection, and then send regular requests. """ - origin = url[:3] + origin = url_to_origin(url) connection = self._get_connection_from_pool(origin) if connection is None: @@ -176,7 +181,10 @@ def _tunnel_request( # CONNECT www.example.org:80 HTTP/1.1 # [proxy-headers] - target = b"%b:%d" % (url[1], url[2]) + if url[2] is None: + target = url[1] + else: + target = b"%b:%d" % (url[1], url[2]) connect_url = self.proxy_origin + (target,) connect_headers = [(b"Host", target), (b"Accept", b"*/*")] connect_headers = merge_headers(connect_headers, self.proxy_headers) diff --git a/httpcore/_types.py b/httpcore/_types.py index e09d4127..d20f0a96 100644 --- a/httpcore/_types.py +++ b/httpcore/_types.py @@ -6,6 +6,6 @@ StrOrBytes = Union[str, bytes] Origin = Tuple[bytes, bytes, int] -URL = Tuple[bytes, bytes, int, bytes] +URL = Tuple[bytes, bytes, Optional[int], bytes] Headers = List[Tuple[bytes, bytes]] TimeoutDict = Dict[str, Optional[float]] diff --git a/httpcore/_utils.py b/httpcore/_utils.py index fe54042c..048f2bf3 100644 --- a/httpcore/_utils.py +++ b/httpcore/_utils.py @@ -2,6 +2,7 @@ import os import sys import typing +from ._types import URL, Origin _LOGGER_INITIALIZED = False TRACE_LOG_LEVEL = 5 @@ -47,3 +48,10 @@ def trace(message: str, *args: typing.Any, **kwargs: typing.Any) -> None: logger.trace = trace # type: ignore return typing.cast(Logger, logger) + + +def url_to_origin(url: URL) -> Origin: + scheme, host, explicit_port = url[:3] + default_port = {b'http': 80, b'https': 443}[scheme] + port = default_port if explicit_port is None else explicit_port + return scheme, host, port From d971abb1e3786ff489c15a397e48e9b72952807c Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 May 2020 10:08:27 +0100 Subject: [PATCH 2/2] Update docstring --- httpcore/_async/http_proxy.py | 4 ++-- httpcore/_sync/http_proxy.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/httpcore/_async/http_proxy.py b/httpcore/_async/http_proxy.py index 1bb93853..629b2cef 100644 --- a/httpcore/_async/http_proxy.py +++ b/httpcore/_async/http_proxy.py @@ -34,8 +34,8 @@ class AsyncHTTPProxy(AsyncConnectionPool): **Parameters:** - * **proxy_origin** - `Tuple[bytes, bytes, int]` - The address of the proxy - service as a 3-tuple of (scheme, host, port). + * **proxy_url** - `Tuple[bytes, bytes, Optional[int], bytes]` - The URL of + the proxy service as a 4-tuple of (scheme, host, port, path). * **proxy_headers** - `Optional[List[Tuple[bytes, bytes]]]` - A list of proxy headers to include. * **proxy_mode** - `str` - A proxy mode to operate in. May be "DEFAULT", diff --git a/httpcore/_sync/http_proxy.py b/httpcore/_sync/http_proxy.py index 71118510..b5edf028 100644 --- a/httpcore/_sync/http_proxy.py +++ b/httpcore/_sync/http_proxy.py @@ -34,8 +34,8 @@ class SyncHTTPProxy(SyncConnectionPool): **Parameters:** - * **proxy_origin** - `Tuple[bytes, bytes, int]` - The address of the proxy - service as a 3-tuple of (scheme, host, port). + * **proxy_url** - `Tuple[bytes, bytes, Optional[int], bytes]` - The URL of + the proxy service as a 4-tuple of (scheme, host, port, path). * **proxy_headers** - `Optional[List[Tuple[bytes, bytes]]]` - A list of proxy headers to include. * **proxy_mode** - `str` - A proxy mode to operate in. May be "DEFAULT",