From 90a59abfb9c525299375dfce6bed04b42af42f84 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 20 Dec 2019 15:14:08 +0100 Subject: [PATCH 001/128] Defaults ws max_size on server to 16MB --- tests/protocols/test_websocket.py | 28 +++++++++++++++++++ uvicorn/config.py | 2 ++ .../protocols/websockets/websockets_impl.py | 2 +- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index a7937d69f..83e899241 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -20,6 +20,7 @@ WebSocketProtocol = None +ONLY_WEBSOCKETPROTOCOL = [p for p in [WebSocketProtocol] if p is not None] WS_PROTOCOLS = [p for p in [WSProtocol, WebSocketProtocol] if p is not None] pytestmark = pytest.mark.skipif( websockets is None, reason="This test needs the websockets module" @@ -463,3 +464,30 @@ async def get_subprotocol(url): accepted_subprotocol = loop.run_until_complete(get_subprotocol(url)) assert accepted_subprotocol == subprotocol loop.close() + + +DEFAULT_MAX_WS_BYTES_PLUS1 = 2 ** 20 + 1 +DEFAULT_MAX_WS_BYTES = 2 ** 20 +MAX_WS_BYTES_MINUS1 = 1024 * 1024 * 16 - 1 + + +@pytest.mark.parametrize("protocol_cls", ONLY_WEBSOCKETPROTOCOL) +def test_send_binary_data_to_server(protocol_cls): + class App(WebSocketResponse): + async def websocket_connect(self, message): + await self.send({"type": "websocket.accept"}) + + async def websocket_receive(self, message): + _bytes = message.get("bytes") + await self.send({"type": "websocket.send", "bytes": _bytes}) + + async def send_text(url): + async with websockets.connect(url, max_size=DEFAULT_MAX_WS_BYTES_PLUS1) as websocket: + await websocket.send(b"\x01"*DEFAULT_MAX_WS_BYTES_PLUS1) + return await websocket.recv() + + with run_server(App, protocol_cls=protocol_cls) as url: + loop = asyncio.new_event_loop() + data = loop.run_until_complete(send_text(url)) + assert data == b"\x01"*DEFAULT_MAX_WS_BYTES_PLUS1 + loop.close() \ No newline at end of file diff --git a/uvicorn/config.py b/uvicorn/config.py index 5bad77d51..d3ccfa61a 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -114,6 +114,7 @@ def __init__( loop="auto", http="auto", ws="auto", + websocket_max_message_size = 16 * 1024 * 1024, lifespan="auto", env_file=None, log_config=LOGGING_CONFIG, @@ -149,6 +150,7 @@ def __init__( self.loop = loop self.http = http self.ws = ws + self.websocket_max_message_size = websocket_max_message_size self.lifespan = lifespan self.log_config = log_config self.log_level = log_level diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index 53f2d4245..a05bd8a7b 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -54,7 +54,7 @@ def __init__(self, config, server_state, _loop=None): self.ws_server = Server() - super().__init__(ws_handler=self.ws_handler, ws_server=self.ws_server) + super().__init__(ws_handler=self.ws_handler, ws_server=self.ws_server, max_size=self.config.websocket_max_message_size) def connection_made(self, transport): self.connections.add(self) From e10829e3f24b76e2e07e3e06302bc80e424f7f8a Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 20 Dec 2019 15:21:34 +0100 Subject: [PATCH 002/128] Renamed test --- tests/protocols/test_websocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 83e899241..fd08bc3e1 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -472,7 +472,7 @@ async def get_subprotocol(url): @pytest.mark.parametrize("protocol_cls", ONLY_WEBSOCKETPROTOCOL) -def test_send_binary_data_to_server(protocol_cls): +def test_send_binary_data_to_server_bigger_than_default(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) From 75e70dc08e13107b9aa13c2c0a737adc651fe62f Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 20 Dec 2019 15:25:22 +0100 Subject: [PATCH 003/128] Lint --- tests/protocols/test_websocket.py | 10 ++++++---- uvicorn/protocols/websockets/websockets_impl.py | 6 +++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index fd08bc3e1..799f646fb 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -482,12 +482,14 @@ async def websocket_receive(self, message): await self.send({"type": "websocket.send", "bytes": _bytes}) async def send_text(url): - async with websockets.connect(url, max_size=DEFAULT_MAX_WS_BYTES_PLUS1) as websocket: - await websocket.send(b"\x01"*DEFAULT_MAX_WS_BYTES_PLUS1) + async with websockets.connect( + url, max_size=DEFAULT_MAX_WS_BYTES_PLUS1 + ) as websocket: + await websocket.send(b"\x01" * DEFAULT_MAX_WS_BYTES_PLUS1) return await websocket.recv() with run_server(App, protocol_cls=protocol_cls) as url: loop = asyncio.new_event_loop() data = loop.run_until_complete(send_text(url)) - assert data == b"\x01"*DEFAULT_MAX_WS_BYTES_PLUS1 - loop.close() \ No newline at end of file + assert data == b"\x01" * DEFAULT_MAX_WS_BYTES_PLUS1 + loop.close() diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index a05bd8a7b..9411fb810 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -54,7 +54,11 @@ def __init__(self, config, server_state, _loop=None): self.ws_server = Server() - super().__init__(ws_handler=self.ws_handler, ws_server=self.ws_server, max_size=self.config.websocket_max_message_size) + super().__init__( + ws_handler=self.ws_handler, + ws_server=self.ws_server, + max_size=self.config.websocket_max_message_size, + ) def connection_made(self, transport): self.connections.add(self) From 8cb0b053857d7f0e4d9b9857a6edcde6c2653462 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 20 Dec 2019 15:27:08 +0100 Subject: [PATCH 004/128] Lint 2 --- uvicorn/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvicorn/config.py b/uvicorn/config.py index d3ccfa61a..316da3163 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -114,7 +114,7 @@ def __init__( loop="auto", http="auto", ws="auto", - websocket_max_message_size = 16 * 1024 * 1024, + websocket_max_message_size=16 * 1024 * 1024, lifespan="auto", env_file=None, log_config=LOGGING_CONFIG, From ebc0d251a0aa1418193054810f7a60413a23d862 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 18:55:15 +0200 Subject: [PATCH 005/128] Renamed to ws_max_size --- uvicorn/config.py | 4 ++-- uvicorn/protocols/websockets/websockets_impl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uvicorn/config.py b/uvicorn/config.py index cf2c36102..8eb1f82b4 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -114,7 +114,7 @@ def __init__( loop="auto", http="auto", ws="auto", - websocket_max_message_size=16 * 1024 * 1024, + ws_max_size=16 * 1024 * 1024, lifespan="auto", env_file=None, log_config=LOGGING_CONFIG, @@ -151,7 +151,7 @@ def __init__( self.loop = loop self.http = http self.ws = ws - self.websocket_max_message_size = websocket_max_message_size + self.ws_max_size = ws_max_size self.lifespan = lifespan self.log_config = log_config self.log_level = log_level diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index 21e477840..2e83217b4 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -57,7 +57,7 @@ def __init__(self, config, server_state, _loop=None): super().__init__( ws_handler=self.ws_handler, ws_server=self.ws_server, - max_size=self.config.websocket_max_message_size, + max_size=self.config.ws_max_size, ) def connection_made(self, transport): From f131cf28feb9880edad8bd2639aaa2368b6c7827 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 20:01:28 +0200 Subject: [PATCH 006/128] Added test for Config --- tests/test_config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index c1f763b41..c37df9838 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -82,3 +82,9 @@ def test_asgi_version(app, expected_interface): config = Config(app=app) config.load() assert config.asgi_version == expected_interface + + +def test_ws_max_size(): + config = Config(app=asgi_app, ws_max_size=1000) + config.load() + assert config.ws_max_size == 1000 \ No newline at end of file From 285da7d397b00a35dd8ec93fcfdf62ad0ef84e72 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 20:08:49 +0200 Subject: [PATCH 007/128] Added click option --- uvicorn/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/uvicorn/main.py b/uvicorn/main.py index 7d63c0f77..c3a9f8652 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -114,6 +114,13 @@ def print_version(ctx, param, value): help="WebSocket protocol implementation.", show_default=True, ) +@click.option( + "--ws-max-size", + type=int, + default=16777216, + help="WebSocket max size message in bytes", + show_default=True, +) @click.option( "--lifespan", type=LIFESPAN_CHOICES, @@ -265,6 +272,7 @@ def main( loop: str, http: str, ws: str, + ws_max_size: int, lifespan: str, interface: str, debug: bool, @@ -302,6 +310,7 @@ def main( "loop": loop, "http": http, "ws": ws, + "ws_max_size": ws_max_size, "lifespan": lifespan, "env_file": env_file, "log_config": LOGGING_CONFIG if log_config is None else log_config, From 509e3139d764ba0ebdf8b373ea1c70361fec9c90 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 20:10:15 +0200 Subject: [PATCH 008/128] Flake8 --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index c37df9838..8b28f1e0a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -87,4 +87,4 @@ def test_asgi_version(app, expected_interface): def test_ws_max_size(): config = Config(app=asgi_app, ws_max_size=1000) config.load() - assert config.ws_max_size == 1000 \ No newline at end of file + assert config.ws_max_size == 1000 From b83d625a04adb6d1b60603bf8f9e8bfc5c467730 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 20:19:50 +0200 Subject: [PATCH 009/128] Updated docs in deployment --- docs/deployment.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index d2e94b7ea..895ae1d52 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -30,26 +30,34 @@ Usage: uvicorn [OPTIONS] APP Options: --host TEXT Bind socket to this host. [default: 127.0.0.1] + --port INTEGER Bind socket to this port. [default: 8000] --uds TEXT Bind to a UNIX domain socket. --fd INTEGER Bind to socket from this file descriptor. --reload Enable auto-reload. --reload-dir TEXT Set reload directories explicitly, instead of using the current working directory. + --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload. - --loop [auto|asyncio|uvloop|iocp] - Event loop implementation. [default: auto] + + --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] + --ws [auto|none|websockets|wsproto] WebSocket protocol implementation. [default: auto] + + --ws-max-size INTEGER WebSocket max size message in bytes + [default: 16777216] + --lifespan [auto|on|off] Lifespan implementation. [default: auto] --interface [auto|asgi3|asgi2|wsgi] Select ASGI3, ASGI2, or WSGI as the application interface. [default: auto] + --env-file PATH Environment configuration file. --log-config PATH Logging configuration file. --log-level [critical|error|warning|info|debug|trace] @@ -60,36 +68,49 @@ Options: Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info. + --forwarded-allow-ips TEXT Comma seperated list of IPs to trust with proxy headers. Defaults to the $FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'. + --root-path TEXT Set the ASGI 'root_path' for applications submounted below a given URL path. + --limit-concurrency INTEGER Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses. + --backlog INTEGER Maximum number of connections to hold in backlog + --limit-max-requests INTEGER Maximum number of requests to service before terminating the process. + --timeout-keep-alive INTEGER Close Keep-Alive connections if no new data is received within this timeout. [default: 5] + --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file --ssl-version INTEGER SSL version to use (see stdlib ssl module's) [default: 2] + --ssl-cert-reqs INTEGER Whether client certificate is required (see stdlib ssl module's) [default: 0] + --ssl-ca-certs TEXT CA certificates file --ssl-ciphers TEXT Ciphers to use (see stdlib ssl module's) [default: TLSv1] + --header TEXT Specify custom default HTTP response headers as a Name:Value pair + + --version Display the uvicorn version and exit. --help Show this message and exit. ``` + See the [settings documentation](settings.md) for more details on the supported options for running uvicorn. ## Running programmatically From a695450cac27b66659434ad35f1ca7d080a66c81 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 20:22:44 +0200 Subject: [PATCH 010/128] Updated docs in index --- docs/index.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index d7d02e1aa..b99b0ab97 100644 --- a/docs/index.md +++ b/docs/index.md @@ -81,26 +81,34 @@ Usage: uvicorn [OPTIONS] APP Options: --host TEXT Bind socket to this host. [default: 127.0.0.1] + --port INTEGER Bind socket to this port. [default: 8000] --uds TEXT Bind to a UNIX domain socket. --fd INTEGER Bind to socket from this file descriptor. --reload Enable auto-reload. --reload-dir TEXT Set reload directories explicitly, instead of using the current working directory. + --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload. - --loop [auto|asyncio|uvloop|iocp] - Event loop implementation. [default: auto] + + --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] + --ws [auto|none|websockets|wsproto] WebSocket protocol implementation. [default: auto] + + --ws-max-size INTEGER WebSocket max size message in bytes + [default: 16777216] + --lifespan [auto|on|off] Lifespan implementation. [default: auto] --interface [auto|asgi3|asgi2|wsgi] Select ASGI3, ASGI2, or WSGI as the application interface. [default: auto] + --env-file PATH Environment configuration file. --log-config PATH Logging configuration file. --log-level [critical|error|warning|info|debug|trace] @@ -111,33 +119,45 @@ Options: Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info. - --forwarded-allow-ips TEXT Comma separated list of IPs to trust with + + --forwarded-allow-ips TEXT Comma seperated list of IPs to trust with proxy headers. Defaults to the $FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'. + --root-path TEXT Set the ASGI 'root_path' for applications submounted below a given URL path. + --limit-concurrency INTEGER Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses. + --backlog INTEGER Maximum number of connections to hold in backlog + --limit-max-requests INTEGER Maximum number of requests to service before terminating the process. + --timeout-keep-alive INTEGER Close Keep-Alive connections if no new data is received within this timeout. [default: 5] + --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file --ssl-version INTEGER SSL version to use (see stdlib ssl module's) [default: 2] + --ssl-cert-reqs INTEGER Whether client certificate is required (see stdlib ssl module's) [default: 0] + --ssl-ca-certs TEXT CA certificates file --ssl-ciphers TEXT Ciphers to use (see stdlib ssl module's) [default: TLSv1] + --header TEXT Specify custom default HTTP response headers as a Name:Value pair + + --version Display the uvicorn version and exit. --help Show this message and exit. ``` From 49e7e6c4fa4478b81dea6ba17376075f336d8113 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 19 May 2020 20:23:00 +0200 Subject: [PATCH 011/128] Added usage note in settings.md --- docs/settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/settings.md b/docs/settings.md index d0a069c59..e96e71086 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -43,6 +43,7 @@ $ pip install uvicorn[watchgodreload] * `--loop ` - Set the event loop implementation. The uvloop implementation provides greater performance, but is not compatible with Windows or PyPy. But you can use IOCP in windows. **Options:** *'auto', 'asyncio', 'uvloop', 'iocp'.* **Default:** *'auto'*. * `--http ` - Set the HTTP protocol implementation. The httptools implementation provides greater performance, but it not compatible with PyPy, and requires compilation on Windows. **Options:** *'auto', 'h11', 'httptools'.* **Default:** *'auto'*. * `--ws ` - Set the WebSockets protocol implementation. Either of the `websockets` and `wsproto` packages are supported. Use `'none'` to deny all websocket requests. **Options:** *'auto', 'none', 'websockets', 'wsproto'.* **Default:** *'auto'*. +* `--ws-max-size ` - Set the WebSockets max message size, in bytes. Please note that this can be used only with the default `websockets` protocol. * `--lifespan ` - Set the Lifespan protocol implementation. **Options:** *'auto', 'on', 'off'.* **Default:** *'auto'*. ## Application Interface From 9b92925a352b9743c5cfcef4a65e74a81a1bad4f Mon Sep 17 00:00:00 2001 From: Peter Baumgartner Date: Wed, 27 May 2020 06:24:30 -0600 Subject: [PATCH 012/128] Document default backlog setting (#682) --- docs/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings.md b/docs/settings.md index d0a069c59..335f2a04c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -71,7 +71,7 @@ connecting IPs in the `forwarded-allow-ips` configuration. * `--limit-concurrency ` - Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses. Useful for ensuring known memory usage patterns even under over-resourced loads. * `--limit-max-requests ` - Maximum number of requests to service before terminating the process. Useful when running together with a process manager, for preventing memory leaks from impacting long-running processes. -* `--backlog ` - Maximum number of connections to hold in backlog. Relevant for heavy incoming traffic. +* `--backlog ` - Maximum number of connections to hold in backlog. Relevant for heavy incoming traffic. **Default:** *2048* ## Timeouts From 77468df7697ec2f2962e77d7a02e0bc22b6f9c79 Mon Sep 17 00:00:00 2001 From: Yezy Ilomo Date: Tue, 9 Jun 2020 13:36:32 +0300 Subject: [PATCH 013/128] Add --app-dir option for running uvicorn from any location (#619) Add '--app-dir' option to specify application directory when running uvicorn from any location Closes #549 --- docs/deployment.md | 3 +++ uvicorn/main.py | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index d2e94b7ea..6ff8144f1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -87,6 +87,9 @@ Options: [default: TLSv1] --header TEXT Specify custom default HTTP response headers as a Name:Value pair + --app-dir TEXT Look for APP in the specified directory, by + adding this to the PYTHONPATH. Defaults to + the current working directory. --help Show this message and exit. ``` diff --git a/uvicorn/main.py b/uvicorn/main.py index 7d63c0f77..25eb54adb 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -256,6 +256,13 @@ def print_version(ctx, param, value): is_eager=True, help="Display the uvicorn version and exit.", ) +@click.option( + "--app-dir", + "app_dir", + default=".", + show_default=True, + help="Look for APP in the specified directory, by adding this to the PYTHONPATH. Defaults to the current working directory.", +) def main( app, host: str, @@ -290,8 +297,9 @@ def main( ssl_ciphers: str, headers: typing.List[str], use_colors: bool, + app_dir: str, ): - sys.path.insert(0, ".") + sys.path.insert(0, app_dir) kwargs = { "app": app, From 6757386460735594b98d6b75e1df6974169909cf Mon Sep 17 00:00:00 2001 From: Bloodielie <49410211+Bloodielie@users.noreply.github.com> Date: Wed, 10 Jun 2020 10:13:06 +0300 Subject: [PATCH 014/128] Removing iocp mentions from documentation (#692) --- docs/deployment.md | 2 +- docs/index.md | 2 +- docs/settings.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 6ff8144f1..ed9605a6e 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -39,7 +39,7 @@ Options: --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload. - --loop [auto|asyncio|uvloop|iocp] + --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] diff --git a/docs/index.md b/docs/index.md index d7d02e1aa..fc3e09591 100644 --- a/docs/index.md +++ b/docs/index.md @@ -90,7 +90,7 @@ Options: --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload. - --loop [auto|asyncio|uvloop|iocp] + --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] diff --git a/docs/settings.md b/docs/settings.md index 335f2a04c..1fc0d028a 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -40,7 +40,7 @@ $ pip install uvicorn[watchgodreload] ## Implementation -* `--loop ` - Set the event loop implementation. The uvloop implementation provides greater performance, but is not compatible with Windows or PyPy. But you can use IOCP in windows. **Options:** *'auto', 'asyncio', 'uvloop', 'iocp'.* **Default:** *'auto'*. +* `--loop ` - Set the event loop implementation. The uvloop implementation provides greater performance, but is not compatible with Windows or PyPy. **Options:** *'auto', 'asyncio', 'uvloop'.* **Default:** *'auto'*. * `--http ` - Set the HTTP protocol implementation. The httptools implementation provides greater performance, but it not compatible with PyPy, and requires compilation on Windows. **Options:** *'auto', 'h11', 'httptools'.* **Default:** *'auto'*. * `--ws ` - Set the WebSockets protocol implementation. Either of the `websockets` and `wsproto` packages are supported. Use `'none'` to deny all websocket requests. **Options:** *'auto', 'none', 'websockets', 'wsproto'.* **Default:** *'auto'*. * `--lifespan ` - Set the Lifespan protocol implementation. **Options:** *'auto', 'on', 'off'.* **Default:** *'auto'*. From f6239169708adcf27433cb8d133e48f8835f0fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Fri, 17 Jul 2020 14:19:07 +0200 Subject: [PATCH 015/128] Release version 0.11.6 (#705) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📝 Update changelog * 🔖 Release version 0.11.6 * 👌 Update/simplify changelog --- CHANGELOG.md | 4 ++++ uvicorn/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 617c366dc..037a940f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## 0.11.6 + +* Fix overriding the root logger. + ## 0.11.5 * Revert "Watch all files, not just .py" due to unexpected side effects. diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 0105b58c9..d7741075e 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.11.5" +__version__ = "0.11.6" __all__ = ["main", "run", "Config", "Server"] From 81f213619d05dd9eb4648271dc1df2cd8b6462d7 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 24 Jul 2020 11:17:39 +0200 Subject: [PATCH 016/128] Brings back windows testing to CI (#685) * 1st pass on adding windows vm for CI * 2nd pass, was testing nothing * No arrays * No arrays 2 * No arrays 3 * No arrays 4 * Removed appveyor PATH export * Using scripts on windows CI, will need to write ps1 scripts * Matrix indent * Location of scripts * Hate CI * Why ps1 are ignored * Adding ps1 scripts, was globally ignored * Chmod +x ps1 * Removing uvloop from win reqs * Added install and test.sh scripts, globally ignored wtf * Align win reqs to old .travis file * Run really on windows ? * Lost in matrix.... * Blind ps1 * Blind ps1 2nd pass * Blind ps1 3rd pass * Blind test.ps1 ! * Blind test.ps1 love * Attempt specifying shell explicitely * Try using bash on windows * Detect os for requirements * Use sh comparison * Live debug * Live debug 2 * More echo * More echo * More echo 2 * OSTYPE only in bash * OSTYPE seems to be msys * Using old names for scripts since we dont need to differentiate now between os * Removed echo leftovers from live debugging and indented back to 4 spaces * Correct set * Removin=g unboud variable check * Set -x before pip only * Update install * Set -e Co-authored-by: Tom Christie --- .github/workflows/test-suite.yml | 9 +++++---- requirements_windows.txt | 19 +++++++++++++++++++ scripts/install | 10 ++++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 requirements_windows.txt diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 2d323e3dd..084e162ea 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -9,13 +9,12 @@ on: jobs: tests: - name: "Python ${{ matrix.python-version }}" - runs-on: "ubuntu-latest" - + name: "Python ${{ matrix.python-version }} ${{ matrix.os }}" + runs-on: "${{ matrix.os }}" strategy: matrix: python-version: ["3.6", "3.7", "3.8"] - + os: [windows-latest, ubuntu-latest] steps: - uses: "actions/checkout@v2" - uses: "actions/setup-python@v1" @@ -23,5 +22,7 @@ jobs: python-version: "${{ matrix.python-version }}" - name: "Install dependencies" run: "scripts/install" + shell: bash - name: "Run tests" run: "scripts/test" + shell: bash diff --git a/requirements_windows.txt b/requirements_windows.txt new file mode 100644 index 000000000..486c96046 --- /dev/null +++ b/requirements_windows.txt @@ -0,0 +1,19 @@ +click +h11 + +# Optional +websockets==8.* +wsproto==0.13.* + +# Testing +autoflake +black +codecov +flake8 +isort +pytest +pytest-cov +requests + +# Efficient debug reload +watchgod>=0.6,<0.7 diff --git a/scripts/install b/scripts/install index 65885a720..b2c9963d2 100755 --- a/scripts/install +++ b/scripts/install @@ -1,9 +1,15 @@ -#!/bin/sh -e +#!/usr/bin/env bash + +set -e # Use the Python executable provided from the `-p` option, or a default. [ "$1" = "-p" ] && PYTHON=$2 || PYTHON="python3" -REQUIREMENTS="requirements.txt" +if [ "$OSTYPE" = "linux-gnu" ]; then + REQUIREMENTS="requirements.txt" +elif [ "$OSTYPE" = "msys" ]; then + REQUIREMENTS="requirements_windows.txt" +fi VENV="venv" set -x From 789e2f135f361d26adda67918fe3f7c4b6ec01b8 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 28 Jul 2020 14:08:17 +0100 Subject: [PATCH 017/128] Disallow invalid header characters (#725) * Disallow invalid header characters * Linting * Fix escape sequence --- uvicorn/protocols/http/httptools_impl.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index 7a60733ce..18161e1e9 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -1,6 +1,7 @@ import asyncio import http import logging +import re import urllib import httptools @@ -13,6 +14,9 @@ is_ssl, ) +HEADER_RE = re.compile(b'[\x00-\x1F\x7F()<>@,;:[]={} \t\\"]') +HEADER_VALUE_RE = re.compile(b"[\x00-\x1F\x7F]") + def _get_status_line(status_code): try: @@ -459,6 +463,11 @@ async def send(self, message): content = [STATUS_LINE[status_code]] for name, value in headers: + if HEADER_RE.search(name): + raise RuntimeError("Invalid HTTP header name.") + if HEADER_VALUE_RE.search(value): + raise RuntimeError("Invalid HTTP header value.") + name = name.lower() if name == b"content-length" and self.chunked_encoding is None: self.expected_content_length = int(value.decode()) From 895807f94ea9a8e588605c12076b7d7517cda503 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 28 Jul 2020 14:08:27 +0100 Subject: [PATCH 018/128] Quote path component before logging (#724) * Quote path component before logging * Linting --- uvicorn/logging.py | 7 ++++--- uvicorn/protocols/utils.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/uvicorn/logging.py b/uvicorn/logging.py index ac910faba..2199e2ce8 100644 --- a/uvicorn/logging.py +++ b/uvicorn/logging.py @@ -1,6 +1,7 @@ import http import logging import sys +import urllib from copy import copy import click @@ -77,14 +78,14 @@ def get_client_addr(self, scope): return "%s:%d" % (client[0], client[1]) def get_path(self, scope): - return scope.get("root_path", "") + scope["path"] + return urllib.parse.quote(scope.get("root_path", "") + scope["path"]) def get_full_path(self, scope): path = scope.get("root_path", "") + scope["path"] query_string = scope.get("query_string", b"").decode("ascii") if query_string: - return path + "?" + query_string - return path + return urllib.parse.quote(path) + "?" + query_string + return urllib.parse.quote(path) def get_status_code(self, record): status_code = record.__dict__["status_code"] diff --git a/uvicorn/protocols/utils.py b/uvicorn/protocols/utils.py index c9347277d..a282c61eb 100644 --- a/uvicorn/protocols/utils.py +++ b/uvicorn/protocols/utils.py @@ -1,4 +1,5 @@ import socket +import urllib def get_remote_addr(transport): @@ -49,7 +50,9 @@ def get_client_addr(scope): def get_path_with_query_string(scope): - path_with_query_string = scope.get("root_path", "") + scope["path"] + path_with_query_string = urllib.parse.quote( + scope.get("root_path", "") + scope["path"] + ) if scope["query_string"]: path_with_query_string = "{}?{}".format( path_with_query_string, scope["query_string"].decode("ascii") From a796e1d4418e8eb917a7856b8530427ef847236c Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 28 Jul 2020 15:09:02 +0200 Subject: [PATCH 019/128] Corrected --proxy-headers client ip/host when using a unix socket (#636) * Added AF_UNIX type socket to get client filled * Removed travis debug * Added travis verbose pytest for windows fail check * Added travis verbose pytest for windows fail check * No more tests output in travis now it's fixed on windows * Useing a top level constant * Lint --- uvicorn/protocols/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/uvicorn/protocols/utils.py b/uvicorn/protocols/utils.py index a282c61eb..b5cd72b50 100644 --- a/uvicorn/protocols/utils.py +++ b/uvicorn/protocols/utils.py @@ -1,6 +1,11 @@ import socket import urllib +if hasattr(socket, "AF_UNIX"): + SUPPORTED_SOCKET_FAMILIES = (socket.AF_INET, socket.AF_INET6, socket.AF_UNIX) +else: + SUPPORTED_SOCKET_FAMILIES = (socket.AF_INET, socket.AF_INET6) + def get_remote_addr(transport): socket_info = transport.get_extra_info("socket") @@ -15,8 +20,9 @@ def get_remote_addr(transport): else: family = socket_info.family - if family in (socket.AF_INET, socket.AF_INET6): + if family in SUPPORTED_SOCKET_FAMILIES: return (str(info[0]), int(info[1])) + return None info = transport.get_extra_info("peername") if info is not None and isinstance(info, (list, tuple)) and len(info) == 2: @@ -29,7 +35,7 @@ def get_local_addr(transport): if socket_info is not None: info = socket_info.getsockname() family = socket_info.family - if family in (socket.AF_INET, socket.AF_INET6): + if family in SUPPORTED_SOCKET_FAMILIES: return (str(info[0]), int(info[1])) return None info = transport.get_extra_info("sockname") From 178bee6ef51c79b296ca70a467f00da20dfae712 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 28 Jul 2020 14:17:52 +0100 Subject: [PATCH 020/128] Version 0.11.7 (#726) --- uvicorn/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index d7741075e..becb78a0e 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.11.6" +__version__ = "0.11.7" __all__ = ["main", "run", "Config", "Server"] From 7163e955190767c687b3c66f24ee557f67a80f53 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 29 Jul 2020 15:10:19 +0100 Subject: [PATCH 021/128] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 037a940f4..5c1d4b6b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## 0.11.7 + +* SECURITY FIX: Prevent sending invalid HTTP header names and values. +* SECURITY FIX: Ensure path value is escaped before logging to the console. + ## 0.11.6 * Fix overriding the root logger. From 0380c4ec24983d9b40eea4645cef3c6991e662ce Mon Sep 17 00:00:00 2001 From: Marco Paolini Date: Thu, 30 Jul 2020 08:50:51 +0100 Subject: [PATCH 022/128] Fix crash when --interface is wsgi (#730) * Fix crash when --interface is wsgi This is a regression introduced in ae0fd316f03dbef926e40216024dfb934417d48d (#597) * Set asgi 3.0 --- tests/test_config.py | 1 + uvicorn/config.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index c1f763b41..ad33707d3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -31,6 +31,7 @@ def test_wsgi_app(): assert isinstance(config.loaded_app, WSGIMiddleware) assert config.interface == "wsgi" + assert config.asgi_version == "3.0" def test_proxy_headers(): diff --git a/uvicorn/config.py b/uvicorn/config.py index fa17cae03..80d9704e3 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -202,7 +202,7 @@ def __init__( @property def asgi_version(self) -> str: - return {"asgi2": "2.0", "asgi3": "3.0"}[self.interface] + return {"asgi2": "2.0", "asgi3": "3.0", "wsgi": "3.0"}[self.interface] @property def is_ssl(self) -> bool: From 8f35b6d9cab37c799d474c6495758f0b5b780006 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Thu, 30 Jul 2020 13:12:03 +0200 Subject: [PATCH 023/128] Update 0.11.7 changelog (#732) --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1d4b6b9..fccc66509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Change Log -## 0.11.7 +## 0.11.7 - 2020-28-07 -* SECURITY FIX: Prevent sending invalid HTTP header names and values. -* SECURITY FIX: Ensure path value is escaped before logging to the console. +* SECURITY FIX: Prevent sending invalid HTTP header names and values. (Pull #725) +* SECURITY FIX: Ensure path value is escaped before logging to the console. (Pull #724) +* Fix `--proxy-headers` client IP and host when using a Unix socket. (Pull #636) ## 0.11.6 From 918722aef95de009fa20de7701286696455f0440 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 31 Jul 2020 12:56:52 +0200 Subject: [PATCH 024/128] Get the request.client under case uvicorn is run with a fd or a unix socket (#729) * Revert "Corrected --proxy-headers client ip/host when using a unix socket (#636)" This reverts commit a796e1d4 * Distinguish case fd/unix socket to return correctly client * Handle windows case * Added test for AF_UNIX socket type Modified MockSocket peername to pass tuples instead of list because socket.getpeername() and socket.getsockname() return tuples * Black * Removed test, black works locally but not in CI.... * Same deal on the server side of things * Test on AF_UNIX only if it is in socket * Simpler handling * Removed debug leftovers --- tests/protocols/test_utils.py | 24 ++++++++++++++++++------ uvicorn/protocols/utils.py | 22 ++++------------------ 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/protocols/test_utils.py b/tests/protocols/test_utils.py index 4f565d4f6..0fff34a5d 100644 --- a/tests/protocols/test_utils.py +++ b/tests/protocols/test_utils.py @@ -29,36 +29,48 @@ def test_get_local_addr_with_socket(): assert get_local_addr(transport) is None transport = MockTransport( - {"socket": MockSocket(family=socket.AF_INET6, sockname=["::1", 123])} + {"socket": MockSocket(family=socket.AF_INET6, sockname=("::1", 123))} ) assert get_local_addr(transport) == ("::1", 123) transport = MockTransport( - {"socket": MockSocket(family=socket.AF_INET, sockname=["123.45.6.7", 123])} + {"socket": MockSocket(family=socket.AF_INET, sockname=("123.45.6.7", 123))} ) assert get_local_addr(transport) == ("123.45.6.7", 123) + if hasattr(socket, "AF_UNIX"): + transport = MockTransport( + {"socket": MockSocket(family=socket.AF_UNIX, sockname=("127.0.0.1", 8000))} + ) + assert get_local_addr(transport) == ("127.0.0.1", 8000) + def test_get_remote_addr_with_socket(): transport = MockTransport({"socket": MockSocket(family=socket.AF_IPX)}) assert get_remote_addr(transport) is None transport = MockTransport( - {"socket": MockSocket(family=socket.AF_INET6, peername=["::1", 123])} + {"socket": MockSocket(family=socket.AF_INET6, peername=("::1", 123))} ) assert get_remote_addr(transport) == ("::1", 123) transport = MockTransport( - {"socket": MockSocket(family=socket.AF_INET, peername=["123.45.6.7", 123])} + {"socket": MockSocket(family=socket.AF_INET, peername=("123.45.6.7", 123))} ) assert get_remote_addr(transport) == ("123.45.6.7", 123) + if hasattr(socket, "AF_UNIX"): + transport = MockTransport( + {"socket": MockSocket(family=socket.AF_UNIX, peername=("127.0.0.1", 8000))} + ) + assert get_remote_addr(transport) == ("127.0.0.1", 8000) + def test_get_local_addr(): transport = MockTransport({"sockname": "path/to/unix-domain-socket"}) assert get_local_addr(transport) is None - transport = MockTransport({"sockname": ["123.45.6.7", 123]}) + transport = MockTransport({"sockname": ("123.45.6.7", 123)}) assert get_local_addr(transport) == ("123.45.6.7", 123) @@ -66,5 +78,5 @@ def test_get_remote_addr(): transport = MockTransport({"peername": None}) assert get_remote_addr(transport) is None - transport = MockTransport({"peername": ["123.45.6.7", 123]}) + transport = MockTransport({"peername": ("123.45.6.7", 123)}) assert get_remote_addr(transport) == ("123.45.6.7", 123) diff --git a/uvicorn/protocols/utils.py b/uvicorn/protocols/utils.py index b5cd72b50..c993f1521 100644 --- a/uvicorn/protocols/utils.py +++ b/uvicorn/protocols/utils.py @@ -1,29 +1,17 @@ -import socket import urllib -if hasattr(socket, "AF_UNIX"): - SUPPORTED_SOCKET_FAMILIES = (socket.AF_INET, socket.AF_INET6, socket.AF_UNIX) -else: - SUPPORTED_SOCKET_FAMILIES = (socket.AF_INET, socket.AF_INET6) - def get_remote_addr(transport): socket_info = transport.get_extra_info("socket") if socket_info is not None: try: info = socket_info.getpeername() + return (str(info[0]), int(info[1])) if isinstance(info, tuple) else None except OSError: # This case appears to inconsistently occur with uvloop # bound to a unix domain socket. - family = None - info = None - else: - family = socket_info.family - - if family in SUPPORTED_SOCKET_FAMILIES: - return (str(info[0]), int(info[1])) + return None - return None info = transport.get_extra_info("peername") if info is not None and isinstance(info, (list, tuple)) and len(info) == 2: return (str(info[0]), int(info[1])) @@ -34,10 +22,8 @@ def get_local_addr(transport): socket_info = transport.get_extra_info("socket") if socket_info is not None: info = socket_info.getsockname() - family = socket_info.family - if family in SUPPORTED_SOCKET_FAMILIES: - return (str(info[0]), int(info[1])) - return None + + return (str(info[0]), int(info[1])) if isinstance(info, tuple) else None info = transport.get_extra_info("sockname") if info is not None and isinstance(info, (list, tuple)) and len(info) == 2: return (str(info[0]), int(info[1])) From 4597b90ffcfb99e44dae6c7d8cc05e1f368e0624 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Fri, 31 Jul 2020 17:47:20 +0200 Subject: [PATCH 025/128] Version 0.11.8 (#731) Co-authored-by: euri10 --- CHANGELOG.md | 5 +++++ uvicorn/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fccc66509..34b74c6c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## 0.11.8 - 2020-07-30 + +* Fix a regression that caused Uvicorn to crash when using `--interface=wsgi`. (Pull #730) +* Fix a regression that caused Uvicorn to crash when using unix domain sockets. (Pull #729) + ## 0.11.7 - 2020-28-07 * SECURITY FIX: Prevent sending invalid HTTP header names and values. (Pull #725) diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index becb78a0e..745b4904e 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.11.7" +__version__ = "0.11.8" __all__ = ["main", "run", "Config", "Server"] From 0abbb6bf596eb991ea40be1e12db0206af288440 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 31 Jul 2020 19:46:41 +0200 Subject: [PATCH 026/128] Added Isuue template based on the one from httpx (#735) --- ISSUE_TEMPLATE/1-question.md | 17 +++++++++ ISSUE_TEMPLATE/2-bug-report.md | 53 +++++++++++++++++++++++++++++ ISSUE_TEMPLATE/3-feature-request.md | 33 ++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 ISSUE_TEMPLATE/1-question.md create mode 100644 ISSUE_TEMPLATE/2-bug-report.md create mode 100644 ISSUE_TEMPLATE/3-feature-request.md diff --git a/ISSUE_TEMPLATE/1-question.md b/ISSUE_TEMPLATE/1-question.md new file mode 100644 index 000000000..27b2b7165 --- /dev/null +++ b/ISSUE_TEMPLATE/1-question.md @@ -0,0 +1,17 @@ +--- +name: Question +about: Ask a question +--- + +### Checklist + + + +- [ ] I searched the [Uvicorn documentation](https://www.uvicorn.org/) but couldn't find what I'm looking for. +- [ ] I looked through similar issues on GitHub, but didn't find anything. +- [ ] I looked up "How to do ... in Uvicorn" on a search engine and didn't find any information. +- [ ] I asked the [community chat](https://gitter.im/encode/community) for help. + +### Question + + \ No newline at end of file diff --git a/ISSUE_TEMPLATE/2-bug-report.md b/ISSUE_TEMPLATE/2-bug-report.md new file mode 100644 index 000000000..a5eccd2b5 --- /dev/null +++ b/ISSUE_TEMPLATE/2-bug-report.md @@ -0,0 +1,53 @@ +--- +name: Bug report +about: Report a bug to help improve this project +--- + +### Checklist + + + +- [ ] The bug is reproducible against the latest release and/or `master`. +- [ ] There are no similar issues or pull requests to fix it yet. + +### Describe the bug + + + +### To reproduce + + + +### Expected behavior + + + +### Actual behavior + + + +### Debugging material + + + +### Environment + +- OS / Python / Uvicorn version: just run `uvicorn --version` +- The exact command you're running uvicorn with, all flags you passed included. If you run it with gunicorn please do the same. If there is a reverse-proxy involved and you cannot reproduce without it please give the minimal config of it to reproduce. + +### Additional context + + \ No newline at end of file diff --git a/ISSUE_TEMPLATE/3-feature-request.md b/ISSUE_TEMPLATE/3-feature-request.md new file mode 100644 index 000000000..d4ef094cb --- /dev/null +++ b/ISSUE_TEMPLATE/3-feature-request.md @@ -0,0 +1,33 @@ +--- +name: Feature request +about: Suggest an idea for this project. +--- + +### Checklist + + + +- [ ] There are no similar issues or pull requests for this yet. +- [ ] I discussed this idea on the [community chat](https://gitter.im/encode/community) and feedback is positive. + +### Is your feature related to a problem? Please describe. + + + +## Describe the solution you would like. + + + +## Describe alternatives you considered + + + +## Additional context + + \ No newline at end of file From 52e1bf429aa9ef65a463108dc28d6328e04bc323 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 4 Aug 2020 07:06:07 +0200 Subject: [PATCH 027/128] Move the dir (#737) --- {ISSUE_TEMPLATE => .github/ISSUE_TEMPLATE}/1-question.md | 0 {ISSUE_TEMPLATE => .github/ISSUE_TEMPLATE}/2-bug-report.md | 0 {ISSUE_TEMPLATE => .github/ISSUE_TEMPLATE}/3-feature-request.md | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {ISSUE_TEMPLATE => .github/ISSUE_TEMPLATE}/1-question.md (100%) rename {ISSUE_TEMPLATE => .github/ISSUE_TEMPLATE}/2-bug-report.md (100%) rename {ISSUE_TEMPLATE => .github/ISSUE_TEMPLATE}/3-feature-request.md (100%) diff --git a/ISSUE_TEMPLATE/1-question.md b/.github/ISSUE_TEMPLATE/1-question.md similarity index 100% rename from ISSUE_TEMPLATE/1-question.md rename to .github/ISSUE_TEMPLATE/1-question.md diff --git a/ISSUE_TEMPLATE/2-bug-report.md b/.github/ISSUE_TEMPLATE/2-bug-report.md similarity index 100% rename from ISSUE_TEMPLATE/2-bug-report.md rename to .github/ISSUE_TEMPLATE/2-bug-report.md diff --git a/ISSUE_TEMPLATE/3-feature-request.md b/.github/ISSUE_TEMPLATE/3-feature-request.md similarity index 100% rename from ISSUE_TEMPLATE/3-feature-request.md rename to .github/ISSUE_TEMPLATE/3-feature-request.md From e77e59612ecae4ac10f9be18f18c47432be7909a Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 6 Aug 2020 09:16:16 +0200 Subject: [PATCH 028/128] Align most of scripts of httpx here (#739) * WIP switch to same layout as httpx * Align scripts on httpx layout * Added scripts, they were ignre by my global gitignore * Mimic httpx workflow better * version is in init file * Added twine and wheel for new scripts * Added mkdocs for win build * Putting the failing overage level to 80 to please windows CI, once this lands we're gonna grind coverage ! * Removed mypy from check * Removed useless marker * Removed check script --- .codecov.yml | 11 --------- .github/workflows/publish.yml | 6 ++++- .github/workflows/test-suite.yml | 9 ++++++++ requirements.txt | 5 ++++ requirements_windows.txt | 9 ++++++++ scripts/build | 13 +++++++++++ scripts/check | 13 +++++++++++ scripts/coverage | 11 +++++++++ scripts/lint | 8 ++++--- scripts/publish | 9 +++----- scripts/test | 18 +++++++++------ setup.cfg | 20 ++++++++++++++++ tests/middleware/test_trace_logging.py | 4 ++-- tests/protocols/test_websocket.py | 11 +++++---- uvicorn/config.py | 2 +- uvicorn/logging.py | 9 ++++++-- uvicorn/main.py | 23 ++++++++++++------- uvicorn/middleware/wsgi.py | 3 ++- uvicorn/protocols/http/h11_impl.py | 3 ++- uvicorn/protocols/http/httptools_impl.py | 3 ++- .../protocols/websockets/websockets_impl.py | 10 ++++++-- uvicorn/protocols/websockets/wsproto_impl.py | 10 ++++++-- 22 files changed, 158 insertions(+), 52 deletions(-) delete mode 100644 .codecov.yml create mode 100755 scripts/build create mode 100755 scripts/check create mode 100755 scripts/coverage create mode 100644 setup.cfg diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index c2336342e..000000000 --- a/.codecov.yml +++ /dev/null @@ -1,11 +0,0 @@ -coverage: - precision: 2 - round: down - range: "80...100" - - status: - project: yes - patch: no - changes: no - -comment: off diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 473cb19b4..a41fd2bf2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,7 +16,11 @@ jobs: - uses: "actions/setup-python@v1" with: python-version: 3.7 - - name: "Publish" + - name: "Install dependencies" + run: "scripts/install" + - name: "Build package & docs" + run: "scripts/build" + - name: "Publish to PyPI & deploy docs" run: "scripts/publish" env: TWINE_USERNAME: __token__ diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 084e162ea..affcbb282 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -23,6 +23,15 @@ jobs: - name: "Install dependencies" run: "scripts/install" shell: bash + - name: "Run linting checks" + run: "scripts/check" + shell: bash + - name: "Build package & docs" + run: "scripts/build" + shell: bash - name: "Run tests" run: "scripts/test" shell: bash + - name: "Enforce coverage" + run: "scripts/coverage" + shell: bash diff --git a/requirements.txt b/requirements.txt index a3592dbdf..102b819b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,10 @@ uvloop>=0.14.0 websockets==8.* wsproto==0.13.* +# Packaging +twine +wheel + # Testing autoflake black @@ -16,6 +20,7 @@ isort pytest pytest-cov requests +seed-isort-config # Documentation mkdocs diff --git a/requirements_windows.txt b/requirements_windows.txt index 486c96046..84c59d7d9 100644 --- a/requirements_windows.txt +++ b/requirements_windows.txt @@ -5,6 +5,10 @@ h11 websockets==8.* wsproto==0.13.* +# Packaging +twine +wheel + # Testing autoflake black @@ -14,6 +18,11 @@ isort pytest pytest-cov requests +seed-isort-config + +# Documentation +mkdocs +mkdocs-material # Efficient debug reload watchgod>=0.6,<0.7 diff --git a/scripts/build b/scripts/build new file mode 100755 index 000000000..1c47d2cc2 --- /dev/null +++ b/scripts/build @@ -0,0 +1,13 @@ +#!/bin/sh -e + +if [ -d 'venv' ] ; then + PREFIX="venv/bin/" +else + PREFIX="" +fi + +set -x + +${PREFIX}python setup.py sdist bdist_wheel +${PREFIX}twine check dist/* +${PREFIX}mkdocs build diff --git a/scripts/check b/scripts/check new file mode 100755 index 000000000..633b5ac16 --- /dev/null +++ b/scripts/check @@ -0,0 +1,13 @@ +#!/bin/sh -e + +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi +export SOURCE_FILES="uvicorn tests" + +set -x + +${PREFIX}black --check --diff --target-version=py36 $SOURCE_FILES +${PREFIX}flake8 $SOURCE_FILES +${PREFIX}isort --check --diff --project=uvicorn $SOURCE_FILES \ No newline at end of file diff --git a/scripts/coverage b/scripts/coverage new file mode 100755 index 000000000..2fb547091 --- /dev/null +++ b/scripts/coverage @@ -0,0 +1,11 @@ +#!/bin/sh -e + +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi +export SOURCE_FILES="uvicorn tests" + +set -x + +${PREFIX}coverage report --show-missing --skip-covered --fail-under=80 \ No newline at end of file diff --git a/scripts/lint b/scripts/lint index 023ea635c..7925c3d7e 100755 --- a/scripts/lint +++ b/scripts/lint @@ -4,9 +4,11 @@ export PREFIX="" if [ -d 'venv' ] ; then export PREFIX="venv/bin/" fi +export SOURCE_FILES="uvicorn tests" set -x -${PREFIX}autoflake --in-place --recursive uvicorn tests -${PREFIX}black uvicorn tests -${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 88 --recursive --apply uvicorn tests +${PREFIX}autoflake --in-place --recursive $SOURCE_FILES +${PREFIX}seed-isort-config --application-directories=uvicorn +${PREFIX}isort --project=uvicorn $SOURCE_FILES +${PREFIX}black --target-version=py36 $SOURCE_FILES diff --git a/scripts/publish b/scripts/publish index b3e9f2dbc..d46377936 100755 --- a/scripts/publish +++ b/scripts/publish @@ -1,5 +1,7 @@ #!/bin/sh -e +VERSION_FILE="uvicorn/__init__.py" + if [ -d 'venv' ] ; then PREFIX="venv/bin/" else @@ -10,7 +12,7 @@ if [ ! -z "$GITHUB_ACTIONS" ]; then git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "GitHub Action" - VERSION=`grep __version__ ./uvicorn/__init__.py | grep -o '[0-9][^"]*'` + VERSION=`grep __version__ ${VERSION_FILE} | grep -o '[0-9][^"]*'` if [ "refs/tags/${VERSION}" != "${GITHUB_REF}" ] ; then echo "GitHub Ref '${GITHUB_REF}' did not match package version '${VERSION}'" @@ -20,10 +22,5 @@ fi set -x -find uvicorn -type f -name "*.py[co]" -delete -find uvicorn -type d -name __pycache__ -delete - -${PREFIX}pip install twine wheel mkdocs mkdocs-material mkautodoc -${PREFIX}python setup.py sdist bdist_wheel ${PREFIX}twine upload dist/* ${PREFIX}mkdocs gh-deploy --force diff --git a/scripts/test b/scripts/test index d871a0e05..f9c991723 100755 --- a/scripts/test +++ b/scripts/test @@ -1,14 +1,18 @@ -#!/bin/sh -e +#!/bin/sh export PREFIX="" if [ -d 'venv' ] ; then export PREFIX="venv/bin/" fi -set -x +set -ex -PYTHONPATH=. ${PREFIX}pytest --ignore venv --cov=uvicorn --cov=tests --cov-report=term-missing ${@} -${PREFIX}coverage html -${PREFIX}autoflake --recursive uvicorn tests -${PREFIX}flake8 uvicorn tests --ignore=W503,E203,E501,E731 -${PREFIX}black uvicorn tests --check +if [ -z $GITHUB_ACTIONS ]; then + scripts/check +fi + +${PREFIX}pytest $@ + +if [ -z $GITHUB_ACTIONS ]; then + scripts/coverage +fi diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..1833312ba --- /dev/null +++ b/setup.cfg @@ -0,0 +1,20 @@ +[flake8] +ignore = W503, E203, B305 +max-line-length = 88 + +[mypy] +disallow_untyped_defs = True +ignore_missing_imports = True + +[mypy-tests.*] +disallow_untyped_defs = False +check_untyped_defs = True + +[tool:isort] +profile = black +combine_as_imports = True +known_first_party = uvicorn,tests +known_third_party = click,does_not_exist,gunicorn,h11,httptools,pytest,requests,setuptools,urllib3,uvloop,watchgod,websockets,wsproto + +[tool:pytest] +addopts = --cov=uvicorn --cov=tests -rxXs diff --git a/tests/middleware/test_trace_logging.py b/tests/middleware/test_trace_logging.py index 5a2e48b11..71ef82398 100644 --- a/tests/middleware/test_trace_logging.py +++ b/tests/middleware/test_trace_logging.py @@ -13,10 +13,10 @@ "disable_existing_loggers": False, "formatters": { "test_formatter_default": { - "format": "[TEST_DEFAULT] %(levelname)-9s %(name)s - %(lineno)d - %(message)s" + "format": "[TEST_DEFAULT] %(levelname)-9s %(name)s - %(lineno)d - %(message)s" # noqa: E501 }, "test_formatter_access": { - "format": "[TEST_ACCESS] %(levelname)-9s %(name)s - %(lineno)d - %(message)s" + "format": "[TEST_ACCESS] %(levelname)-9s %(name)s - %(lineno)d - %(message)s" # noqa: E501 }, "test_formatter_asgi": { "format": "[TEST_ASGI] %(levelname)-9s %(name)s - %(lineno)d - %(message)s" diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 2e0855226..98445de2a 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -14,6 +14,7 @@ try: import websockets + from uvicorn.protocols.websockets.websockets_impl import WebSocketProtocol except ImportError: # pragma: nocover websockets = None @@ -78,8 +79,8 @@ def run_server(app, protocol_cls, path="/"): @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) def test_invalid_upgrade(protocol_cls): - - app = lambda scope: None + def app(scope): + return None with run_server(app, protocol_cls=protocol_cls) as url: url = url.replace("ws://", "http://") @@ -95,8 +96,10 @@ def test_invalid_upgrade(protocol_cls): "missing sec-websocket-key header", "missing sec-websocket-version header", # websockets "missing or empty sec-websocket-key header", # wsproto - "failed to open a websocket connection: missing sec-websocket-key header", - "failed to open a websocket connection: missing or empty sec-websocket-key header", + "failed to open a websocket connection: missing " + "sec-websocket-key header", + "failed to open a websocket connection: missing or empty " + "sec-websocket-key header", ] diff --git a/uvicorn/config.py b/uvicorn/config.py index 80d9704e3..e4c6e9426 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -67,7 +67,7 @@ }, "access": { "()": "uvicorn.logging.AccessFormatter", - "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', + "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', # noqa: E501 }, }, "handlers": { diff --git a/uvicorn/logging.py b/uvicorn/logging.py index 2199e2ce8..1efaf92ba 100644 --- a/uvicorn/logging.py +++ b/uvicorn/logging.py @@ -37,7 +37,9 @@ def __init__(self, fmt=None, datefmt=None, style="%", use_colors=None): super().__init__(fmt=fmt, datefmt=datefmt, style=style) def color_level_name(self, level_name, level_no): - default = lambda level_name: str(level_name) + def default(level_name): + return str(level_name) + func = self.level_name_colors.get(level_no, default) return func(level_name) @@ -96,7 +98,10 @@ def get_status_code(self, record): status_and_phrase = "%s %s" % (status_code, status_phrase) if self.use_colors: - default = lambda code: status_and_phrase + + def default(code): + return status_and_phrase + func = self.status_code_colours.get(status_code // 100, default) return func(status_and_phrase) return status_and_phrase diff --git a/uvicorn/main.py b/uvicorn/main.py index 25eb54adb..0111353b1 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -85,13 +85,15 @@ def print_version(ctx, param, value): "--reload-dir", "reload_dirs", multiple=True, - help="Set reload directories explicitly, instead of using the current working directory.", + help="Set reload directories explicitly, instead of using the current working" + " directory.", ) @click.option( "--workers", default=None, type=int, - help="Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload.", + help="Number of worker processes. Defaults to the $WEB_CONCURRENCY environment" + " variable if available. Not valid with --reload.", ) @click.option( "--loop", @@ -165,13 +167,15 @@ def print_version(ctx, param, value): "--proxy-headers/--no-proxy-headers", is_flag=True, default=True, - help="Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info.", + help="Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to " + "populate remote address info.", ) @click.option( "--forwarded-allow-ips", type=str, default=None, - help="Comma seperated list of IPs to trust with proxy headers. Defaults to the $FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'.", + help="Comma seperated list of IPs to trust with proxy headers. Defaults to" + " the $FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'.", ) @click.option( "--root-path", @@ -183,7 +187,8 @@ def print_version(ctx, param, value): "--limit-concurrency", type=int, default=None, - help="Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses.", + help="Maximum number of concurrent connections or tasks to allow, before issuing" + " HTTP 503 responses.", ) @click.option( "--backlog", @@ -261,7 +266,8 @@ def print_version(ctx, param, value): "app_dir", default=".", show_default=True, - help="Look for APP in the specified directory, by adding this to the PYTHONPATH. Defaults to the current working directory.", + help="Look for APP in the specified directory, by adding this to the PYTHONPATH." + " Defaults to the current working directory.", ) def main( app, @@ -345,8 +351,9 @@ def run(app, **kwargs): if (config.reload or config.workers > 1) and not isinstance(app, str): logger = logging.getLogger("uvicorn.error") - logger.warn( - "You must pass the application as an import string to enable 'reload' or 'workers'." + logger.warning( + "You must pass the application as an import string to enable 'reload' or " + "'workers'." ) sys.exit(1) diff --git a/uvicorn/middleware/wsgi.py b/uvicorn/middleware/wsgi.py index d713ff8e5..e5346ddcf 100644 --- a/uvicorn/middleware/wsgi.py +++ b/uvicorn/middleware/wsgi.py @@ -44,7 +44,8 @@ def build_environ(scope, message, body): corrected_name = "CONTENT_TYPE" else: corrected_name = "HTTP_%s" % name.upper().replace("-", "_") - # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in case + # HTTPbis say only ASCII chars are allowed in headers, but we latin1 + # just in case value = value.decode("latin1") if corrected_name in environ: value = environ[corrected_name] + "," + value diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index 5e15d0512..fb70038ef 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -336,7 +336,8 @@ def resume_writing(self): def timeout_keep_alive_handler(self): """ - Called on a keep-alive connection if no new data is received after a short delay. + Called on a keep-alive connection if no new data is received after a short + delay. """ if not self.transport.is_closing(): event = h11.ConnectionClosed() diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index 18161e1e9..1642b7abb 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -338,7 +338,8 @@ def resume_writing(self): def timeout_keep_alive_handler(self): """ - Called on a keep-alive connection if no new data is received after a short delay. + Called on a keep-alive connection if no new data is received after a short + delay. """ if not self.transport.is_closing(): self.transport.close() diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index 37e2cb784..407674cf0 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -199,7 +199,10 @@ async def asgi_send(self, message): self.closed_event.set() else: - msg = "Expected ASGI message 'websocket.accept' or 'websocket.close', but got '%s'." + msg = ( + "Expected ASGI message 'websocket.accept' or 'websocket.close', " + "but got '%s'." + ) raise RuntimeError(msg % message_type) elif not self.closed_event.is_set(): @@ -217,7 +220,10 @@ async def asgi_send(self, message): self.closed_event.set() else: - msg = "Expected ASGI message 'websocket.send' or 'websocket.close', but got '%s'." + msg = ( + "Expected ASGI message 'websocket.send' or 'websocket.close'," + " but got '%s'." + ) raise RuntimeError(msg % message_type) else: diff --git a/uvicorn/protocols/websockets/wsproto_impl.py b/uvicorn/protocols/websockets/wsproto_impl.py index 307ab9f29..d712963bd 100644 --- a/uvicorn/protocols/websockets/wsproto_impl.py +++ b/uvicorn/protocols/websockets/wsproto_impl.py @@ -263,7 +263,10 @@ async def send(self, message): self.transport.close() else: - msg = "Expected ASGI message 'websocket.accept' or 'websocket.close', but got '%s'." + msg = ( + "Expected ASGI message 'websocket.accept' or 'websocket.close', " + "but got '%s'." + ) raise RuntimeError(msg % message_type) elif not self.close_sent: @@ -285,7 +288,10 @@ async def send(self, message): self.transport.close() else: - msg = "Expected ASGI message 'websocket.send' or 'websocket.close', but got '%s'." + msg = ( + "Expected ASGI message 'websocket.send' or 'websocket.close'," + " but got '%s'." + ) raise RuntimeError(msg % message_type) else: From 11f9a71f2c131e9aeb644824c41035b26fc06d83 Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 6 Aug 2020 14:42:05 +0200 Subject: [PATCH 029/128] Install script back to where it was and same as httpx (#746) Removed conditional requirements.txt for windonws from script > using pip markers --- requirements.txt | 4 ++-- requirements_windows.txt | 28 ---------------------------- scripts/install | 12 +++--------- 3 files changed, 5 insertions(+), 39 deletions(-) delete mode 100644 requirements_windows.txt diff --git a/requirements.txt b/requirements.txt index 102b819b9..9536034cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ click h11 # Optional -httptools -uvloop>=0.14.0 +httptools; sys_platform != 'win32' +uvloop>=0.14.0; sys_platform != 'win32' websockets==8.* wsproto==0.13.* diff --git a/requirements_windows.txt b/requirements_windows.txt deleted file mode 100644 index 84c59d7d9..000000000 --- a/requirements_windows.txt +++ /dev/null @@ -1,28 +0,0 @@ -click -h11 - -# Optional -websockets==8.* -wsproto==0.13.* - -# Packaging -twine -wheel - -# Testing -autoflake -black -codecov -flake8 -isort -pytest -pytest-cov -requests -seed-isort-config - -# Documentation -mkdocs -mkdocs-material - -# Efficient debug reload -watchgod>=0.6,<0.7 diff --git a/scripts/install b/scripts/install index b2c9963d2..09f9e3782 100755 --- a/scripts/install +++ b/scripts/install @@ -1,15 +1,9 @@ -#!/usr/bin/env bash - -set -e +#!/bin/sh -e # Use the Python executable provided from the `-p` option, or a default. [ "$1" = "-p" ] && PYTHON=$2 || PYTHON="python3" -if [ "$OSTYPE" = "linux-gnu" ]; then - REQUIREMENTS="requirements.txt" -elif [ "$OSTYPE" = "msys" ]; then - REQUIREMENTS="requirements_windows.txt" -fi +REQUIREMENTS="requirements.txt" VENV="venv" set -x @@ -22,4 +16,4 @@ else fi "$PIP" install -r "$REQUIREMENTS" -"$PIP" install -e . +"$PIP" install -e . \ No newline at end of file From 5fa99a11d6e07367cd559a4fc8694ce537796558 Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 10 Aug 2020 18:49:30 +0200 Subject: [PATCH 030/128] Use optional package installs (#666) * Make dependencies optional resolves #219 * Tweak install options and update installation instructions in readme * update to latest conclusion * rename default -> standard * Incoroporate other reqs * Added setup.py to black test, previous merge made me remove all double quotes in it * Added setup.py to lint script to match test script * Lint setup.py * Removed leftover from merge that was incorrect * Trying to be more explicit about the 2 install options * Spelling * Newlines * Better wording * Websockets vs wsproto more relevant in extras Co-authored-by: Carl George Co-authored-by: Almar Klein --- README.md | 20 ++++++++++++++++++++ requirements.txt | 5 +++-- setup.py | 23 +++++++++++++++-------- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c8d86fb9e..fe9924557 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,26 @@ Install using `pip`: $ pip install uvicorn ``` +This will install uvicorn with minimal (pure Python) dependencies. + +```shell +$ pip install uvicorn[standard] +``` + +This will install uvicorn with "Cython-based" dependencies (where possible) and other "optional extras". + +In this context, "Cython-based" means the following: + +- the event loop `uvloop` will be installed and used if possible. +- the http protocol will be handled by `httptools` if possible. + +Moreover, "optional extras" means that: + +- the websocket protocol will be handled by `websockets` (should you want to use `wsproto` you'd need to install it manually) if possible. +- the `--reloader` flag in development mode will use `watchgod`. +- windows users will have `colorama` installed for the colored logs. +- `python-dotenv` will be install should you want to use the `--env-file` option. + Create an application, in `example.py`: ```python diff --git a/requirements.txt b/requirements.txt index 9536034cf..a9b66873f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +# Minimal click h11 @@ -6,6 +7,8 @@ httptools; sys_platform != 'win32' uvloop>=0.14.0; sys_platform != 'win32' websockets==8.* wsproto==0.13.* +watchgod>=0.6,<0.7 +python_dotenv==0.13.* # Packaging twine @@ -26,5 +29,3 @@ seed-isort-config mkdocs mkdocs-material -# Efficient debug reload -watchgod>=0.6,<0.7 diff --git a/setup.py b/setup.py index ef6504c19..1ea8a534c 100755 --- a/setup.py +++ b/setup.py @@ -34,21 +34,28 @@ def get_packages(package): ] -env_marker = ( +env_marker_cpython = ( "sys_platform != 'win32'" " and sys_platform != 'cygwin'" " and platform_python_implementation != 'PyPy'" ) -requirements = [ +env_marker_win = "sys_platform == 'win32'" + + +minimal_requirements = [ "click==7.*", "h11>=0.8,<0.10", - "websockets==8.*", - "httptools==0.1.* ;" + env_marker, - "uvloop>=0.14.0 ;" + env_marker, ] -extras_require = {"watchgodreload": ["watchgod>=0.6,<0.7"]} +extra_requirements = [ + "websockets==8.*", + "httptools==0.1.* ;" + env_marker_cpython, + "uvloop>=0.14.0 ;" + env_marker_cpython, + "colorama>=0.4.*;" + env_marker_win, + "watchgod>=0.6,<0.7", + "python-dotenv==0.13.*", +] setup( @@ -62,8 +69,8 @@ def get_packages(package): author="Tom Christie", author_email="tom@tomchristie.com", packages=get_packages("uvicorn"), - install_requires=requirements, - extras_require=extras_require, + install_requires=minimal_requirements, + extras_require={"standard": extra_requirements}, include_package_data=True, classifiers=[ "Development Status :: 4 - Beta", From a9c37cc41ff6698e9ffa8dceedf6b9d223d0b42e Mon Sep 17 00:00:00 2001 From: Peter Morrow Date: Tue, 11 Aug 2020 09:40:07 -0700 Subject: [PATCH 031/128] Fixes bug where --log-config disables uvicorn loggers (#512) * Fixes bug where --log-config disables uvicorn loggers * Runs black to fix formatting issue --- uvicorn/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uvicorn/config.py b/uvicorn/config.py index e4c6e9426..4e046de94 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -222,7 +222,9 @@ def configure_logging(self): ] = self.use_colors logging.config.dictConfig(self.log_config) else: - logging.config.fileConfig(self.log_config) + logging.config.fileConfig( + self.log_config, disable_existing_loggers=False + ) if self.log_level is not None: if isinstance(self.log_level, str): From fbce393fd34038a589bba02e759b0f051f52ad84 Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 13 Aug 2020 12:35:40 +0200 Subject: [PATCH 032/128] Upgrade wsproto to 0.15.0 (#750) * Relaxed wsproto pin * Pinned to latest wsproto --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a9b66873f..9dc75f013 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ h11 httptools; sys_platform != 'win32' uvloop>=0.14.0; sys_platform != 'win32' websockets==8.* -wsproto==0.13.* +wsproto==0.15.* watchgod>=0.6,<0.7 python_dotenv==0.13.* From 8150c3ebe0a8c9721497c3b455cf8759f0e94afa Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 15 Aug 2020 07:57:27 -0700 Subject: [PATCH 033/128] Added asgi dict to the lifespan scope (#754) * Added asgi dict to the lifespan scope * Added lifespan scope tests --- tests/test_lifespan.py | 37 +++++++++++++++++++++++++++++++++++++ uvicorn/lifespan/on.py | 5 ++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py index 15823d09c..d72269ab1 100644 --- a/tests/test_lifespan.py +++ b/tests/test_lifespan.py @@ -144,3 +144,40 @@ async def test(): loop = asyncio.new_event_loop() loop.run_until_complete(test()) + + +@pytest.mark.parametrize("mode", ("auto", "on")) +def test_lifespan_scope_asgi3app(mode): + async def asgi3app(scope, receive, send): + assert scope == {"version": "3.0", "spec_version": "2.0"} + + async def test(): + config = Config(app=asgi3app, lifespan=mode) + lifespan = LifespanOn(config) + + await lifespan.startup() + await lifespan.shutdown() + + loop = asyncio.new_event_loop() + loop.run_until_complete(test()) + + +@pytest.mark.parametrize("mode", ("auto", "on")) +def test_lifespan_scope_asgi2app(mode): + def asgi2app(scope): + assert scope == {"version": "2.0", "spec_version": "2.0"} + + async def asgi(receive, send): + pass + + return asgi + + async def test(): + config = Config(app=asgi2app, lifespan=mode) + lifespan = LifespanOn(config) + + await lifespan.startup() + await lifespan.shutdown() + + loop = asyncio.new_event_loop() + loop.run_until_complete(test()) diff --git a/uvicorn/lifespan/on.py b/uvicorn/lifespan/on.py index 6797b8919..4deb8c883 100644 --- a/uvicorn/lifespan/on.py +++ b/uvicorn/lifespan/on.py @@ -44,7 +44,10 @@ async def shutdown(self): async def main(self): try: app = self.config.loaded_app - scope = {"type": "lifespan"} + scope = { + "type": "lifespan", + "asgi": {"version": self.config.asgi_version, "spec_version": "2.0"}, + } await app(scope, self.receive, self.send) except BaseException as exc: self.asgi = None From a9a46e87d8ba41f08b3a401b7b0cc572333a7193 Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 15 Aug 2020 23:54:36 -0700 Subject: [PATCH 034/128] Mypy lifespan (#751) * Progressive mypy, we begin by lifespan * Lifespan typed * We need mypy to use mypy ...:) * Trying to fix windows not having uvloop in _types.py * Using setup.cfg to specify files * Try removing the try catch, will it please CI ? * Removed all unecessary types * Put back the try catch on 3.8 - * Trying to simplify the < 3.8 for Literal and TypedDict * After lookig back WSend was off * Tightens http messages and put asgi ref in their docstring * Tightens ws messages and put asgi ref in their docstring * Blacked * Using lifespan messages and scope * Force typing_extensions for users under 3.8 * Simplified types * Simplified types again ! --- requirements.txt | 1 + scripts/check | 1 + setup.cfg | 2 ++ setup.py | 3 ++- uvicorn/_types.py | 31 +++++++++++++++++++++++++++++++ uvicorn/lifespan/off.py | 9 ++++++--- uvicorn/lifespan/on.py | 20 ++++++++++++-------- 7 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 uvicorn/_types.py diff --git a/requirements.txt b/requirements.txt index 9dc75f013..34a91e3fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ pytest pytest-cov requests seed-isort-config +mypy # Documentation mkdocs diff --git a/scripts/check b/scripts/check index 633b5ac16..1f2190183 100755 --- a/scripts/check +++ b/scripts/check @@ -10,4 +10,5 @@ set -x ${PREFIX}black --check --diff --target-version=py36 $SOURCE_FILES ${PREFIX}flake8 $SOURCE_FILES +${PREFIX}mypy ${PREFIX}isort --check --diff --project=uvicorn $SOURCE_FILES \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 1833312ba..748336210 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,8 @@ max-line-length = 88 [mypy] disallow_untyped_defs = True ignore_missing_imports = True +follow_imports = silent +files = uvicorn/lifespan,tests/test_lifespan.py [mypy-tests.*] disallow_untyped_defs = False diff --git a/setup.py b/setup.py index 1ea8a534c..865b91cc9 100755 --- a/setup.py +++ b/setup.py @@ -41,11 +41,12 @@ def get_packages(package): ) env_marker_win = "sys_platform == 'win32'" - +env_marker_below_38 = "python_version < '3.8'" minimal_requirements = [ "click==7.*", "h11>=0.8,<0.10", + "typing-extensions;" + env_marker_below_38 ] extra_requirements = [ diff --git a/uvicorn/_types.py b/uvicorn/_types.py new file mode 100644 index 000000000..d7967c244 --- /dev/null +++ b/uvicorn/_types.py @@ -0,0 +1,31 @@ +import sys +from typing import Optional + +if sys.version_info < (3, 8): + from typing_extensions import Literal, TypedDict +else: + from typing import Literal, TypedDict + + +class ASGISpecInfo(TypedDict): + version: str + spec_version: Optional[Literal["2.0", "2.1"]] + + +class LifespanScope(TypedDict): + type: Literal["lifespan"] + asgi: ASGISpecInfo + + +class LifespanReceiveMessage(TypedDict): + type: Literal["lifespan.startup", "lifespan.shutdown"] + + +class LifespanSendMessage(TypedDict): + type: Literal[ + "lifespan.startup.complete", + "lifespan.startup.failed", + "lifespan.shutdown.complete", + "lifespan.shutdown.failed", + ] + message: Optional[str] diff --git a/uvicorn/lifespan/off.py b/uvicorn/lifespan/off.py index c23895a54..7ec961b5f 100644 --- a/uvicorn/lifespan/off.py +++ b/uvicorn/lifespan/off.py @@ -1,9 +1,12 @@ +from uvicorn import Config + + class LifespanOff: - def __init__(self, config): + def __init__(self, config: Config) -> None: self.should_exit = False - async def startup(self): + async def startup(self) -> None: pass - async def shutdown(self): + async def shutdown(self) -> None: pass diff --git a/uvicorn/lifespan/on.py b/uvicorn/lifespan/on.py index 4deb8c883..0f0644fef 100644 --- a/uvicorn/lifespan/on.py +++ b/uvicorn/lifespan/on.py @@ -1,11 +1,15 @@ import asyncio import logging +from asyncio import Queue + +from uvicorn import Config +from uvicorn._types import LifespanReceiveMessage, LifespanScope, LifespanSendMessage STATE_TRANSITION_ERROR = "Got invalid state transition on lifespan protocol." class LifespanOn: - def __init__(self, config): + def __init__(self, config: Config) -> None: if not config.loaded: config.load() @@ -13,12 +17,12 @@ def __init__(self, config): self.logger = logging.getLogger("uvicorn.error") self.startup_event = asyncio.Event() self.shutdown_event = asyncio.Event() - self.receive_queue = asyncio.Queue() + self.receive_queue: "Queue[LifespanReceiveMessage]" = asyncio.Queue() self.error_occured = False self.startup_failed = False self.should_exit = False - async def startup(self): + async def startup(self) -> None: self.logger.info("Waiting for application startup.") loop = asyncio.get_event_loop() @@ -33,7 +37,7 @@ async def startup(self): else: self.logger.info("Application startup complete.") - async def shutdown(self): + async def shutdown(self) -> None: if self.error_occured: return self.logger.info("Waiting for application shutdown.") @@ -41,10 +45,10 @@ async def shutdown(self): await self.shutdown_event.wait() self.logger.info("Application shutdown complete.") - async def main(self): + async def main(self) -> None: try: app = self.config.loaded_app - scope = { + scope: LifespanScope = { "type": "lifespan", "asgi": {"version": self.config.asgi_version, "spec_version": "2.0"}, } @@ -64,7 +68,7 @@ async def main(self): self.startup_event.set() self.shutdown_event.set() - async def send(self, message): + async def send(self, message: LifespanSendMessage) -> None: assert message["type"] in ( "lifespan.startup.complete", "lifespan.startup.failed", @@ -89,5 +93,5 @@ async def send(self, message): assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR self.shutdown_event.set() - async def receive(self): + async def receive(self) -> LifespanReceiveMessage: return await self.receive_queue.get() From 093a1f7c1ef7e30ec0b8b63618a4d5ee59f323aa Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Mon, 17 Aug 2020 23:36:12 -0700 Subject: [PATCH 035/128] Allow .json or .yaml --log-config files (#665) * Allow .json or .yaml --log-config files This adds support for `.json` and `.yaml` config files when running `uvicorn` from the CLI. This allows clients to define how they wish to deal with existing loggers (#511, #512) by providing`disable_existing_loggers` in their config file (the last item described in [this section](https://docs.python.org/3/library/logging.config.html#dictionary-schema-details)). Furthermore, it addresses the desire to allow users to replicate the default hard-coded `LOGGING_CONFIG` in their own configs and tweak it as necessary, something that is not currently possible for clients that wish to run their apps from the CLI. * Add ids to parametrized config test case * Fix black formatting and don't use .ini extension in the fileConfig() test * Remove item from extras_require * Address PR feedback --- README.md | 5 ++- docs/deployment.md | 1 + docs/index.md | 1 + docs/settings.md | 6 ++- requirements.txt | 2 + setup.py | 3 +- tests/test_config.py | 96 +++++++++++++++++++++++++++++++++++++++++++- uvicorn/config.py | 19 +++++++++ 8 files changed, 126 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fe9924557..bdfd33d3c 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,9 @@ Moreover, "optional extras" means that: - the websocket protocol will be handled by `websockets` (should you want to use `wsproto` you'd need to install it manually) if possible. - the `--reloader` flag in development mode will use `watchgod`. - windows users will have `colorama` installed for the colored logs. -- `python-dotenv` will be install should you want to use the `--env-file` option. - +- `python-dotenv` will be installed should you want to use the `--env-file` option. +- `PyYAML` will be installed to allow you to provide a `.yaml` file to `--log-config`, if desired. + Create an application, in `example.py`: ```python diff --git a/docs/deployment.md b/docs/deployment.md index ed9605a6e..3cf9a9034 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -52,6 +52,7 @@ Options: application interface. [default: auto] --env-file PATH Environment configuration file. --log-config PATH Logging configuration file. + Supported formats (.ini, .json, .yaml) --log-level [critical|error|warning|info|debug|trace] Log level. [default: info] --access-log / --no-access-log Enable/Disable access log. diff --git a/docs/index.md b/docs/index.md index fc3e09591..0991cbc65 100644 --- a/docs/index.md +++ b/docs/index.md @@ -103,6 +103,7 @@ Options: application interface. [default: auto] --env-file PATH Environment configuration file. --log-config PATH Logging configuration file. + Supported formats (.ini, .json, .yaml) --log-level [critical|error|warning|info|debug|trace] Log level. [default: info] --access-log / --no-access-log Enable/Disable access log. diff --git a/docs/settings.md b/docs/settings.md index 1fc0d028a..617ac63ef 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -33,10 +33,12 @@ $ pip install uvicorn[watchgodreload] ## Logging -* `--log-config ` - Logging configuration file. +* `--log-config ` - Logging configuration file. **Options:** *`dictConfig()` formats: .json, .yaml*. Any other format will be processed with `fileConfig()`. Set the `formatters.default.use_colors` and `formatters.access.use_colors` values to override the auto-detected behavior. + * If you wish to use a YAML file for your logging config, you will need to include PyYAML as a dependency for your project or install uvicorn with the `[standard]` optional extras. * `--log-level ` - Set the log level. **Options:** *'critical', 'error', 'warning', 'info', 'debug', 'trace'.* **Default:** *'info'*. * `--no-access-log` - Disable access log only, without changing log level. -* `--use-colors / --no-use-colors` - Enable / disable colorized formatting of the log records, in case this is not set it will be auto-detected. +* `--use-colors / --no-use-colors` - Enable / disable colorized formatting of the log records, in case this is not set it will be auto-detected. This option is ignored if the `--log-config` CLI option is used. + ## Implementation diff --git a/requirements.txt b/requirements.txt index 34a91e3fa..4a6d08473 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ websockets==8.* wsproto==0.15.* watchgod>=0.6,<0.7 python_dotenv==0.13.* +PyYAML>=5.1 # Packaging twine @@ -22,6 +23,7 @@ flake8 isort pytest pytest-cov +pytest-mock requests seed-isort-config mypy diff --git a/setup.py b/setup.py index 865b91cc9..2978d31bf 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def get_packages(package): minimal_requirements = [ "click==7.*", "h11>=0.8,<0.10", - "typing-extensions;" + env_marker_below_38 + "typing-extensions;" + env_marker_below_38, ] extra_requirements = [ @@ -56,6 +56,7 @@ def get_packages(package): "colorama>=0.4.*;" + env_marker_win, "watchgod>=0.6,<0.7", "python-dotenv==0.13.*", + "PyYAML>=5.1", ] diff --git a/tests/test_config.py b/tests/test_config.py index ad33707d3..336ff5b8b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,14 +1,37 @@ +import json import socket +from copy import deepcopy import pytest +import yaml from uvicorn import protocols -from uvicorn.config import Config +from uvicorn.config import LOGGING_CONFIG, Config from uvicorn.middleware.debug import DebugMiddleware from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware from uvicorn.middleware.wsgi import WSGIMiddleware +@pytest.fixture +def mocked_logging_config_module(mocker): + return mocker.patch("logging.config") + + +@pytest.fixture +def logging_config(): + return deepcopy(LOGGING_CONFIG) + + +@pytest.fixture +def json_logging_config(logging_config): + return json.dumps(logging_config) + + +@pytest.fixture +def yaml_logging_config(logging_config): + return yaml.dump(logging_config) + + async def asgi_app(): pass # pragma: nocover @@ -77,9 +100,78 @@ async def asgi(receive, send): @pytest.mark.parametrize( - "app, expected_interface", [(asgi_app, "3.0",), (asgi2_app, "2.0",)] + "app, expected_interface", [(asgi_app, "3.0"), (asgi2_app, "2.0")] ) def test_asgi_version(app, expected_interface): config = Config(app=app) config.load() assert config.asgi_version == expected_interface + + +@pytest.mark.parametrize( + "use_colors, expected", + [ + pytest.param(None, None, id="use_colors_not_provided"), + pytest.param(True, True, id="use_colors_enabled"), + pytest.param(False, False, id="use_colors_disabled"), + pytest.param("invalid", False, id="use_colors_invalid_value"), + ], +) +def test_log_config_default(mocked_logging_config_module, use_colors, expected): + """ + Test that one can specify the use_colors option when using the default logging + config. + """ + config = Config(app=asgi_app, use_colors=use_colors) + config.load() + + mocked_logging_config_module.dictConfig.assert_called_once_with(LOGGING_CONFIG) + + ((provided_dict_config,), _,) = mocked_logging_config_module.dictConfig.call_args + assert provided_dict_config["formatters"]["default"]["use_colors"] == expected + + +def test_log_config_json( + mocked_logging_config_module, logging_config, json_logging_config, mocker +): + """ + Test that one can load a json config from disk. + """ + mocked_open = mocker.patch( + "uvicorn.config.open", mocker.mock_open(read_data=json_logging_config) + ) + + config = Config(app=asgi_app, log_config="log_config.json") + config.load() + + mocked_open.assert_called_once_with("log_config.json") + mocked_logging_config_module.dictConfig.assert_called_once_with(logging_config) + + +def test_log_config_yaml( + mocked_logging_config_module, logging_config, yaml_logging_config, mocker +): + """ + Test that one can load a yaml config from disk. + """ + mocked_open = mocker.patch( + "uvicorn.config.open", mocker.mock_open(read_data=yaml_logging_config) + ) + + config = Config(app=asgi_app, log_config="log_config.yaml") + config.load() + + mocked_open.assert_called_once_with("log_config.yaml") + mocked_logging_config_module.dictConfig.assert_called_once_with(logging_config) + + +def test_log_config_file(mocked_logging_config_module): + """ + Test that one can load a configparser config from disk. + """ + config = Config(app=asgi_app, log_config="log_config") + config.load() + + mocked_logging_config_module.fileConfig.assert_called_once_with( + "log_config", disable_existing_loggers=False + ) diff --git a/uvicorn/config.py b/uvicorn/config.py index 4e046de94..f8012fc50 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -1,5 +1,6 @@ import asyncio import inspect +import json import logging import logging.config import os @@ -10,6 +11,14 @@ import click +try: + import yaml +except ImportError: + # If the code below that depends on yaml is exercised, it will raise a NameError. + # Install the PyYAML package or the uvicorn[standard] optional dependencies to + # enable this functionality. + pass + from uvicorn.importer import ImportFromStringError, import_from_string from uvicorn.middleware.asgi2 import ASGI2Middleware from uvicorn.middleware.debug import DebugMiddleware @@ -221,7 +230,17 @@ def configure_logging(self): "use_colors" ] = self.use_colors logging.config.dictConfig(self.log_config) + elif self.log_config.endswith(".json"): + with open(self.log_config) as file: + loaded_config = json.load(file) + logging.config.dictConfig(loaded_config) + elif self.log_config.endswith(".yaml"): + with open(self.log_config) as file: + loaded_config = yaml.safe_load(file) + logging.config.dictConfig(loaded_config) else: + # See the note about fileConfig() here: + # https://docs.python.org/3/library/logging.config.html#configuration-file-format logging.config.fileConfig( self.log_config, disable_existing_loggers=False ) From dd3b842db72aeed991cc1ae1227707fb4d9c81d6 Mon Sep 17 00:00:00 2001 From: Aber Date: Thu, 27 Aug 2020 16:53:12 +0800 Subject: [PATCH 036/128] Fix terminate error in windows (#744) --- uvicorn/supervisors/basereload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uvicorn/supervisors/basereload.py b/uvicorn/supervisors/basereload.py index ffdf8cb9e..db2bf85c6 100644 --- a/uvicorn/supervisors/basereload.py +++ b/uvicorn/supervisors/basereload.py @@ -62,7 +62,8 @@ def startup(self): def restart(self): self.mtimes = {} - os.kill(self.process.pid, signal.SIGTERM) + + self.process.terminate() self.process.join() self.process = get_subprocess( From aa046f561629d9d8ff6b9419e0deddd114f766dd Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 27 Aug 2020 03:17:42 -0700 Subject: [PATCH 037/128] Black 20 is the new black (#769) * Black 20 is the new black * Revert "Black 20 is the new black" * Trailing subtelty --- setup.cfg | 2 +- tests/protocols/test_http.py | 2 +- tests/test_config.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 748336210..21103a6a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ check_untyped_defs = True profile = black combine_as_imports = True known_first_party = uvicorn,tests -known_third_party = click,does_not_exist,gunicorn,h11,httptools,pytest,requests,setuptools,urllib3,uvloop,watchgod,websockets,wsproto +known_third_party = click,does_not_exist,gunicorn,h11,httptools,pytest,requests,setuptools,urllib3,uvloop,watchgod,websockets,wsproto,yaml [tool:pytest] addopts = --cov=uvicorn --cov=tests -rxXs diff --git a/tests/protocols/test_http.py b/tests/protocols/test_http.py index 891a259f2..298714850 100644 --- a/tests/protocols/test_http.py +++ b/tests/protocols/test_http.py @@ -696,7 +696,7 @@ async def asgi(receive, send): @pytest.mark.parametrize("asgi2or3_app, expected_scopes", asgi_scope_data) @pytest.mark.parametrize("protocol_cls", HTTP_PROTOCOLS) def test_scopes(asgi2or3_app, expected_scopes, protocol_cls): - protocol = get_connected_protocol(asgi2or3_app, protocol_cls,) + protocol = get_connected_protocol(asgi2or3_app, protocol_cls) protocol.data_received(SIMPLE_GET_REQUEST) protocol.loop.run_one() assert expected_scopes == protocol.scope.get("asgi") diff --git a/tests/test_config.py b/tests/test_config.py index 336ff5b8b..8c70de66f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -127,7 +127,7 @@ def test_log_config_default(mocked_logging_config_module, use_colors, expected): mocked_logging_config_module.dictConfig.assert_called_once_with(LOGGING_CONFIG) - ((provided_dict_config,), _,) = mocked_logging_config_module.dictConfig.call_args + (provided_dict_config,), _ = mocked_logging_config_module.dictConfig.call_args assert provided_dict_config["formatters"]["default"]["use_colors"] == expected From 2e22b850540de6a4bd2b6b488452bf0e38da7797 Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 27 Aug 2020 08:55:04 -0700 Subject: [PATCH 038/128] Drop deprecated isort-seed-config (#770) --- requirements.txt | 1 - scripts/lint | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4a6d08473..2ec5c2c5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,6 @@ pytest pytest-cov pytest-mock requests -seed-isort-config mypy # Documentation diff --git a/scripts/lint b/scripts/lint index 7925c3d7e..795d5d01a 100755 --- a/scripts/lint +++ b/scripts/lint @@ -9,6 +9,5 @@ export SOURCE_FILES="uvicorn tests" set -x ${PREFIX}autoflake --in-place --recursive $SOURCE_FILES -${PREFIX}seed-isort-config --application-directories=uvicorn ${PREFIX}isort --project=uvicorn $SOURCE_FILES ${PREFIX}black --target-version=py36 $SOURCE_FILES From b782588de9caa5989c53f307ed4dfdf171a2f5de Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 27 Aug 2020 18:32:52 +0200 Subject: [PATCH 039/128] chore: add config for pointing to Gitter chat (#768) * chore: add config for pointing to Gitter chat * chore: remove question issue template --- .github/ISSUE_TEMPLATE/1-question.md | 17 ----------------- .github/ISSUE_TEMPLATE/config.yml | 7 +++++++ 2 files changed, 7 insertions(+), 17 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/1-question.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/1-question.md b/.github/ISSUE_TEMPLATE/1-question.md deleted file mode 100644 index 27b2b7165..000000000 --- a/.github/ISSUE_TEMPLATE/1-question.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Question -about: Ask a question ---- - -### Checklist - - - -- [ ] I searched the [Uvicorn documentation](https://www.uvicorn.org/) but couldn't find what I'm looking for. -- [ ] I looked through similar issues on GitHub, but didn't find anything. -- [ ] I looked up "How to do ... in Uvicorn" on a search engine and didn't find any information. -- [ ] I asked the [community chat](https://gitter.im/encode/community) for help. - -### Question - - \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..2ad6e8e27 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +# Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser +blank_issues_enabled: true +contact_links: +- name: Question + url: https://gitter.im/encode/community + about: > + Ask a question From df81b1684493ad97e8ba3fa323cc329089880a7c Mon Sep 17 00:00:00 2001 From: Weiliang Li Date: Fri, 28 Aug 2020 15:36:47 +0900 Subject: [PATCH 040/128] Dont set log level for root logger (#767) --- uvicorn/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/uvicorn/config.py b/uvicorn/config.py index f8012fc50..3299d4c09 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -250,7 +250,6 @@ def configure_logging(self): log_level = LOG_LEVELS[self.log_level] else: log_level = self.log_level - logging.getLogger("").setLevel(log_level) logging.getLogger("uvicorn.error").setLevel(log_level) logging.getLogger("uvicorn.access").setLevel(log_level) logging.getLogger("uvicorn.asgi").setLevel(log_level) From ff4af12d6902bc9d535fe2a948d1df3ffa02b0d3 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 28 Aug 2020 01:15:35 -0700 Subject: [PATCH 041/128] Revert "Improve shutdown robustness when using `--reload` or multiprocessing (#620)" (#756) This reverts commit fdb89e9c --- uvicorn/subprocess.py | 11 ----------- uvicorn/supervisors/basereload.py | 9 +-------- uvicorn/supervisors/multiprocess.py | 9 +-------- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/uvicorn/subprocess.py b/uvicorn/subprocess.py index ded8801ef..735a51353 100644 --- a/uvicorn/subprocess.py +++ b/uvicorn/subprocess.py @@ -4,7 +4,6 @@ """ import multiprocessing import os -import signal import sys multiprocessing.allow_connection_pickling() @@ -60,13 +59,3 @@ def subprocess_started(config, target, sockets, stdin_fileno): # Now we can call into `Server.run(sockets=sockets)` target(sockets=sockets) - - -def shutdown_subprocess(pid): - """ - Helper to attempt cleanly shutting down a subprocess. May fail with an exception. - - * pid - Process identifier. - """ - os.kill(pid, signal.SIGINT) - os.waitpid(pid, 0) diff --git a/uvicorn/supervisors/basereload.py b/uvicorn/supervisors/basereload.py index db2bf85c6..f234bb647 100644 --- a/uvicorn/supervisors/basereload.py +++ b/uvicorn/supervisors/basereload.py @@ -5,7 +5,7 @@ import click -from uvicorn.subprocess import get_subprocess, shutdown_subprocess +from uvicorn.subprocess import get_subprocess HANDLED_SIGNALS = ( signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. @@ -22,19 +22,12 @@ def __init__(self, config, target, sockets): self.sockets = sockets self.should_exit = threading.Event() self.pid = os.getpid() - self.process = None self.reloader_name = None def signal_handler(self, sig, frame): """ A signal handler that is registered with the parent process. """ - if self.process is not None: - try: - shutdown_subprocess(self.process.pid) - except Exception as exc: - logger.error(f"Could not stop child process {self.process.pid}: {exc}") - self.should_exit.set() def run(self): diff --git a/uvicorn/supervisors/multiprocess.py b/uvicorn/supervisors/multiprocess.py index d6b4de8cf..94f7238d3 100644 --- a/uvicorn/supervisors/multiprocess.py +++ b/uvicorn/supervisors/multiprocess.py @@ -5,7 +5,7 @@ import click -from uvicorn.subprocess import get_subprocess, shutdown_subprocess +from uvicorn.subprocess import get_subprocess HANDLED_SIGNALS = ( signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. @@ -28,13 +28,6 @@ def signal_handler(self, sig, frame): """ A signal handler that is registered with the parent process. """ - - for process in self.processes: - try: - shutdown_subprocess(process.pid) - except Exception as exc: - logger.error(f"Could not stop child process {process.pid}: {exc}") - self.should_exit.set() def run(self): From 54d729ccc1638180a51e6ab600c4724e6424b048 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Fri, 28 Aug 2020 16:47:49 +0200 Subject: [PATCH 042/128] Upgrade maximum h11 dependency version to 0.10 (#772) The h11 Changelog for 0.10 only include those items: - Drop support for Python 3.4. - Support Python 3.8. - Make error messages returned by match failures less ambiguous (#98). --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2978d31bf..f3599b9fb 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_packages(package): minimal_requirements = [ "click==7.*", - "h11>=0.8,<0.10", + "h11>=0.8,<0.11", "typing-extensions;" + env_marker_below_38, ] From 980100274470e1b04dce5e5c8d50ce487aab3145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Pito=C5=84?= Date: Mon, 28 Sep 2020 10:54:31 +0200 Subject: [PATCH 043/128] Make reload delay configurable (#774) * Make reload delay configurable * Rebase and fix line length --- uvicorn/config.py | 2 ++ uvicorn/main.py | 10 ++++++++++ uvicorn/supervisors/basereload.py | 3 ++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/uvicorn/config.py b/uvicorn/config.py index 3299d4c09..ef65214e1 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -133,6 +133,7 @@ def __init__( debug=False, reload=False, reload_dirs=None, + reload_delay=None, workers=None, proxy_headers=True, forwarded_allow_ips=None, @@ -167,6 +168,7 @@ def __init__( self.interface = interface self.debug = debug self.reload = reload + self.reload_delay = reload_delay or 0.25 self.workers = workers or 1 self.proxy_headers = proxy_headers self.root_path = root_path diff --git a/uvicorn/main.py b/uvicorn/main.py index 0111353b1..adcc37d38 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -88,6 +88,14 @@ def print_version(ctx, param, value): help="Set reload directories explicitly, instead of using the current working" " directory.", ) +@click.option( + "--reload-delay", + type=float, + default=0.25, + show_default=True, + help="Delay between previous and next check if application needs to be." + " Defaults to 0.25s.", +) @click.option( "--workers", default=None, @@ -283,6 +291,7 @@ def main( debug: bool, reload: bool, reload_dirs: typing.List[str], + reload_delay: float, workers: int, env_file: str, log_config: str, @@ -325,6 +334,7 @@ def main( "debug": debug, "reload": reload, "reload_dirs": reload_dirs if reload_dirs else None, + "reload_delay": reload_delay, "workers": workers, "proxy_headers": proxy_headers, "forwarded_allow_ips": forwarded_allow_ips, diff --git a/uvicorn/supervisors/basereload.py b/uvicorn/supervisors/basereload.py index f234bb647..a2afc563c 100644 --- a/uvicorn/supervisors/basereload.py +++ b/uvicorn/supervisors/basereload.py @@ -32,9 +32,10 @@ def signal_handler(self, sig, frame): def run(self): self.startup() - while not self.should_exit.wait(0.25): + while not self.should_exit.wait(self.config.reload_delay): if self.should_restart(): self.restart() + self.shutdown() def startup(self): From 342c68d36502ce0849f165836744365ad0462c80 Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 28 Sep 2020 02:57:00 -0700 Subject: [PATCH 044/128] Version 0.12.0 (#771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Push new nersion * Added one entry * Update CHANGELOG.md Co-authored-by: Rafał Pitoń * Update CHANGELOG.md Co-authored-by: Rafał Pitoń * Modified date * More bump * Added reload delay PR Co-authored-by: Rafał Pitoń --- CHANGELOG.md | 18 ++++++++++++++++++ uvicorn/__init__.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34b74c6c5..202d17508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Change Log +## 0.12.0 - 2020-09-28 + +### Added +- Make reload delay configurable (#774) 9/28/20 98010027 +- Upgrade maximum h11 dependency version to 0.10 (#772) 8/28/20 54d729cc +- Allow .json or .yaml --log-config files (#665) 8/18/20 093a1f7c +- Add ASGI dict to the lifespan scope (#754) 8/15/20 8150c3eb +- Upgrade wsproto to 0.15.0 (#750) 8/13/20 fbce393f +- Use optional package installs (#666) 8/10/20 5fa99a11 + +### Changed +- Dont set log level for root logger (#767) 8/28/20 df81b168 + +### Fixed +- Revert "Improve shutdown robustness when using `--reload` or multiprocessing (#620)" (#756) 8/28/20 ff4af12d +- Fix terminate error in windows (#744) 8/27/20 dd3b842d +- Fix bug where --log-config disables uvicorn loggers (#512) 8/11/20 a9c37cc4 + ## 0.11.8 - 2020-07-30 * Fix a regression that caused Uvicorn to crash when using `--interface=wsgi`. (Pull #730) diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 745b4904e..8ede93c4f 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.11.8" +__version__ = "0.12.0" __all__ = ["main", "run", "Config", "Server"] From d5fa9d84f0706aee9b238a2676ff26944da14519 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Tue, 29 Sep 2020 10:13:45 +0200 Subject: [PATCH 045/128] Move package editable install to requirements.txt (#790) * Move package editable install to requirements.txt * Add wsproto --- requirements.txt | 12 ++---------- scripts/install | 1 - 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2ec5c2c5f..fb7dde77c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,7 @@ -# Minimal -click -h11 +-e .[standard] -# Optional -httptools; sys_platform != 'win32' -uvloop>=0.14.0; sys_platform != 'win32' -websockets==8.* +# Explicit optionals wsproto==0.15.* -watchgod>=0.6,<0.7 -python_dotenv==0.13.* -PyYAML>=5.1 # Packaging twine diff --git a/scripts/install b/scripts/install index 09f9e3782..d80157ddd 100755 --- a/scripts/install +++ b/scripts/install @@ -16,4 +16,3 @@ else fi "$PIP" install -r "$REQUIREMENTS" -"$PIP" install -e . \ No newline at end of file From 70ebcfdf8aa14c9395c7f58a62f9347582979cf1 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 29 Sep 2020 09:16:28 +0100 Subject: [PATCH 046/128] Get docs/index.md in sync with README.md (#784) --- docs/index.md | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/docs/index.md b/docs/index.md index 0991cbc65..79f038b66 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,11 +19,11 @@ # Introduction -Uvicorn is a lightning-fast ASGI server, built on [uvloop][uvloop] and [httptools][httptools]. +Uvicorn is a lightning-fast ASGI server implementation, using [uvloop][uvloop] and [httptools][httptools]. Until recently Python has lacked a minimal low-level server/application interface for -asyncio frameworks. The [ASGI specification][asgi] fills this gap, and means we're now able to start building -a common set of tooling usable across all asyncio frameworks. +asyncio frameworks. The [ASGI specification][asgi] fills this gap, and means we're now able to +start building a common set of tooling usable across all asyncio frameworks. ASGI should help enable an ecosystem of Python web frameworks that are highly competitive against Node and Go in terms of achieving high throughput in IO-bound contexts. It also provides support for HTTP/2 and @@ -31,29 +31,47 @@ WebSockets, which cannot be handled by WSGI. Uvicorn currently supports HTTP/1.1 and WebSockets. Support for HTTP/2 is planned. ---- - ## Quickstart -Requirements: Python 3.5, 3.6, 3.7, 3.8 - Install using `pip`: -``` +```shell $ pip install uvicorn ``` +This will install uvicorn with minimal (pure Python) dependencies. + +```shell +$ pip install uvicorn[standard] +``` + +This will install uvicorn with "Cython-based" dependencies (where possible) and other "optional extras". + +In this context, "Cython-based" means the following: + +- the event loop `uvloop` will be installed and used if possible. +- the http protocol will be handled by `httptools` if possible. + +Moreover, "optional extras" means that: + +- the websocket protocol will be handled by `websockets` (should you want to use `wsproto` you'd need to install it manually) if possible. +- the `--reloader` flag in development mode will use `watchgod`. +- windows users will have `colorama` installed for the colored logs. +- `python-dotenv` will be installed should you want to use the `--env-file` option. +- `PyYAML` will be installed to allow you to provide a `.yaml` file to `--log-config`, if desired. + Create an application, in `example.py`: ```python async def app(scope, receive, send): assert scope['type'] == 'http' + await send({ 'type': 'http.response.start', 'status': 200, 'headers': [ [b'content-type', b'text/plain'], - ] + ], }) await send({ 'type': 'http.response.body', @@ -63,7 +81,7 @@ async def app(scope, receive, send): Run the server: -``` +```shell $ uvicorn example:app ``` From bbf19c66c30e01bd12fac6ea55f64a40b79adbe8 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 29 Sep 2020 01:24:05 -0700 Subject: [PATCH 047/128] Pinning h11 and python-dotenv to min versions (#789) * Removed max pin and chhanged to >=min * Revert all but h11 and pyton-dotenv --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f3599b9fb..2fa4bb1dd 100755 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_packages(package): minimal_requirements = [ "click==7.*", - "h11>=0.8,<0.11", + "h11>=0.8", "typing-extensions;" + env_marker_below_38, ] @@ -55,7 +55,7 @@ def get_packages(package): "uvloop>=0.14.0 ;" + env_marker_cpython, "colorama>=0.4.*;" + env_marker_win, "watchgod>=0.6,<0.7", - "python-dotenv==0.13.*", + "python-dotenv>=0.13.*", "PyYAML>=5.1", ] From e2b750647f8f367852a379aa5f669425bfb45559 Mon Sep 17 00:00:00 2001 From: Viktor Ahlqvist Date: Tue, 29 Sep 2020 13:35:29 +0200 Subject: [PATCH 048/128] Improve changelog by pointing out breaking changes (#792) --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202d17508..4dab18493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ ### Changed - Dont set log level for root logger (#767) 8/28/20 df81b168 +- Uvicorn no longer ships extra dependencies `uvloop`, `websockets` and + `httptools` as default. To install these dependencies use + `uvicorn[standard]`. ### Fixed - Revert "Improve shutdown robustness when using `--reload` or multiprocessing (#620)" (#756) 8/28/20 ff4af12d From cd00516c6c5d22e83ceb1d786f80a340ab388e6f Mon Sep 17 00:00:00 2001 From: euri10 Date: Wed, 30 Sep 2020 06:13:26 -0700 Subject: [PATCH 049/128] Version 0.12.1 (#794) --- CHANGELOG.md | 9 +++++++++ uvicorn/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dab18493..fe7e79f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log +## 0.12.1 - 2020-09-30 + +### Changed +- Pinning h11 and python-dotenv to min versions (#789) 9/29/20 bbf19c66 +- Get docs/index.md in sync with README.md (#784) 9/29/20 70ebcfdf + +### Fixed +- Improve changelog by pointing out breaking changes (#792) 9/29/20 e2b75064 + ## 0.12.0 - 2020-09-28 ### Added diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 8ede93c4f..7d0702b3b 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.12.0" +__version__ = "0.12.1" __all__ = ["main", "run", "Config", "Server"] From a504c56963fb0d4c9add33fe3d1cb3fc25d82fdb Mon Sep 17 00:00:00 2001 From: Peter Law Date: Sun, 4 Oct 2020 12:16:18 +0100 Subject: [PATCH 050/128] Note the need to configure trusted "ips" when using unix sockets (#796) It is unfortunately non-obvious that when using an unix socket, clients connecting to that socket are not trusted as a source of headers for proxying to the underlying application. Fixes https://github.com/encode/uvicorn/issues/713 --- docs/deployment.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deployment.md b/docs/deployment.md index 3cf9a9034..299e67564 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -204,6 +204,7 @@ Using Nginx as a proxy in front of your Uvicorn processes may not be neccessary, In managed environments such as `Heroku`, you wont typically need to configure Nginx, as your server processes will already be running behind load balancing proxies. The recommended configuration for proxying from Nginx is to use a UNIX domain socket between Nginx and whatever the process manager that is being used to run Uvicorn. +Note that when doing this you will need run Uvicorn with `--forwarded-allow-ips='*'` to ensure that the domain socket is trusted as a source from which to proxy headers. When fronting the application with a proxy server you want to make sure that the proxy sets headers to ensure that application can properly determine the client address of the incoming connection, and if the connection was over `http` or `https`. From 08fd05590d089bea5db3eb0f8ed202c1f25730e6 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 6 Oct 2020 19:01:43 +0200 Subject: [PATCH 051/128] Added python 3.9 support (#804) --- .github/workflows/publish.yml | 2 +- .github/workflows/test-suite.yml | 4 ++-- setup.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a41fd2bf2..b290d6e1a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: "actions/checkout@v2" - - uses: "actions/setup-python@v1" + - uses: "actions/setup-python@v2" with: python-version: 3.7 - name: "Install dependencies" diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index affcbb282..b8a82b639 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -13,11 +13,11 @@ jobs: runs-on: "${{ matrix.os }}" strategy: matrix: - python-version: ["3.6", "3.7", "3.8"] + python-version: ["3.6", "3.7", "3.8", "3.9"] os: [windows-latest, ubuntu-latest] steps: - uses: "actions/checkout@v2" - - uses: "actions/setup-python@v1" + - uses: "actions/setup-python@v2" with: python-version: "${{ matrix.python-version }}" - name: "Install dependencies" diff --git a/setup.py b/setup.py index 2fa4bb1dd..ab7a751da 100755 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ def get_packages(package): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ], From b468950ec24d2361f8372e077d3074dac87812f0 Mon Sep 17 00:00:00 2001 From: Maksim Rakitin Date: Tue, 6 Oct 2020 13:05:03 -0400 Subject: [PATCH 052/128] Support .yml log config files (#799) * Support .yml log config files * TST: add a test for .yml file extension * TST: parametrize tests; remove .coverage.* and add this to .gitignore * STY: fix the style with the linter * Revert .gitignore --- tests/test_config.py | 11 ++++++++--- uvicorn/config.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 8c70de66f..d8052dc2e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -148,8 +148,13 @@ def test_log_config_json( mocked_logging_config_module.dictConfig.assert_called_once_with(logging_config) +@pytest.mark.parametrize("config_filename", ["log_config.yml", "log_config.yaml"]) def test_log_config_yaml( - mocked_logging_config_module, logging_config, yaml_logging_config, mocker + mocked_logging_config_module, + logging_config, + yaml_logging_config, + mocker, + config_filename, ): """ Test that one can load a yaml config from disk. @@ -158,10 +163,10 @@ def test_log_config_yaml( "uvicorn.config.open", mocker.mock_open(read_data=yaml_logging_config) ) - config = Config(app=asgi_app, log_config="log_config.yaml") + config = Config(app=asgi_app, log_config=config_filename) config.load() - mocked_open.assert_called_once_with("log_config.yaml") + mocked_open.assert_called_once_with(config_filename) mocked_logging_config_module.dictConfig.assert_called_once_with(logging_config) diff --git a/uvicorn/config.py b/uvicorn/config.py index ef65214e1..944317e6c 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -236,7 +236,7 @@ def configure_logging(self): with open(self.log_config) as file: loaded_config = json.load(file) logging.config.dictConfig(loaded_config) - elif self.log_config.endswith(".yaml"): + elif self.log_config.endswith((".yaml", ".yml")): with open(self.log_config) as file: loaded_config = yaml.safe_load(file) logging.config.dictConfig(loaded_config) From d755fe850d2bf740c5dd091e091e154457634c5f Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 8 Oct 2020 10:11:52 +0200 Subject: [PATCH 053/128] Drop pytest-cov and uses vanilla coverage (#809) * Removed pytest-cov * Removed pytest-cov 2 * Added vanilla coverage * Include > source indeed * Unused export --- requirements.txt | 3 +-- scripts/test | 2 +- setup.cfg | 6 +++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index fb7dde77c..3758ca584 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,14 +10,13 @@ wheel # Testing autoflake black -codecov flake8 isort pytest -pytest-cov pytest-mock requests mypy +coverage # Documentation mkdocs diff --git a/scripts/test b/scripts/test index f9c991723..9d3568790 100755 --- a/scripts/test +++ b/scripts/test @@ -11,7 +11,7 @@ if [ -z $GITHUB_ACTIONS ]; then scripts/check fi -${PREFIX}pytest $@ +${PREFIX}coverage run --debug config -m pytest if [ -z $GITHUB_ACTIONS ]; then scripts/coverage diff --git a/setup.cfg b/setup.cfg index 21103a6a4..04ee1d0b4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,4 +19,8 @@ known_first_party = uvicorn,tests known_third_party = click,does_not_exist,gunicorn,h11,httptools,pytest,requests,setuptools,urllib3,uvloop,watchgod,websockets,wsproto,yaml [tool:pytest] -addopts = --cov=uvicorn --cov=tests -rxXs +addopts = -rxXs + +[coverage:run] +omit = venv/* +include = uvicorn/*, tests/* From 103167a0557df277d63efb66cec182731d624bf0 Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 12 Oct 2020 08:53:56 +0200 Subject: [PATCH 054/128] Sharing socket across workers on windows (#802) --- uvicorn/main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uvicorn/main.py b/uvicorn/main.py index adcc37d38..487d29e15 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -452,8 +452,19 @@ async def startup(self, sockets=None): if sockets is not None: # Explicitly passed a list of open sockets. # We use this when the server is run from a Gunicorn worker. + + def _share_socket(sock: socket) -> socket: + # Windows requires the socket be explicitly shared across + # multiple workers (processes). + from socket import fromshare # type: ignore + + sock_data = sock.share(os.getpid()) # type: ignore + return fromshare(sock_data) + self.servers = [] for sock in sockets: + if config.workers > 1 and platform.system() == "Windows": + sock = _share_socket(sock) server = await loop.create_server( create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog ) From 6873289921f07ce85b672ab3636eb960703bf062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Collopy?= Date: Mon, 12 Oct 2020 09:59:35 +0200 Subject: [PATCH 055/128] Added cli suport for headers containing colon (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some http headers need to have colon in its values, such as: ``` Content-Security-Policy: default-src 'self'; script-src https://example.com; ``` Passing this value though the client used to raise an error, because this header would be parsed as: ``` ["Content-Security-Policy", "default-src 'self'; script-src https", "//example.com;"] ``` And could no be unpached as key, value pair. This commit limits the spliting on colon to its first occuence in the string, enabling us to pass a value containing semicolon in the cli: ``` $ uvicorn main:app --header 'Content-Security-Policy:default-src *:8000' ``` Co-authored-by: Rômulo Collopy --- tests/test_cli.py | 30 ++++++++++++++++++++++++++++++ uvicorn/main.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 000000000..706ed13a9 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,30 @@ +import importlib +from unittest import mock + +from click.testing import CliRunner + +from uvicorn.main import main as cli + +HEADERS = "Content-Security-Policy:default-src 'self'; script-src https://example.com" +main = importlib.import_module("uvicorn.main") + + +def test_cli_headers(): + runner = CliRunner() + + with mock.patch.object(main, "run") as mock_run: + result = runner.invoke(cli, ["tests.test_cli:App", "--header", HEADERS]) + + assert result.output == "" + assert result.exit_code == 0 + mock_run.assert_called_once() + assert mock_run.call_args[1]["headers"] == [ + [ + "Content-Security-Policy", + "default-src 'self'; script-src https://example.com", + ] + ] + + +class App: + pass diff --git a/uvicorn/main.py b/uvicorn/main.py index 487d29e15..e243ebb03 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -349,7 +349,7 @@ def main( "ssl_cert_reqs": ssl_cert_reqs, "ssl_ca_certs": ssl_ca_certs, "ssl_ciphers": ssl_ciphers, - "headers": list([header.split(":") for header in headers]), + "headers": list([header.split(":", 1) for header in headers]), "use_colors": use_colors, } run(**kwargs) From 23d274de872c38b1956cb53f32ecb0ac3a3fd55d Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 12 Oct 2020 10:01:03 +0200 Subject: [PATCH 056/128] Refactor ssl tooling using trustme (#807) * !st pass using trustme * 2nd pass with more precise fixtures * 3rd pass * Forgot dev requirements.txt * test updated + black * Black * Added test to run uvicorn with a chain certificate ! * Added config test as well when using combined certificate and pk --- requirements.txt | 1 + tests/conftest.py | 92 +++++++----------------- tests/supervisors/test_watchgodreload.py | 4 +- tests/test_config.py | 19 ++++- tests/test_ssl.py | 40 +++++++++-- 5 files changed, 83 insertions(+), 73 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3758ca584..fd61c1dee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ pytest pytest-mock requests mypy +trustme coverage # Documentation diff --git a/tests/conftest.py b/tests/conftest.py index 8d743e3fb..f396cb415 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,72 +1,34 @@ import pytest +import trustme -CERTIFICATE = b"""-----BEGIN CERTIFICATE----- -MIIEaDCCAtCgAwIBAgIRAPeU748qfVOTZJ7rj5DupbowDQYJKoZIhvcNAQELBQAw -fTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMSkwJwYDVQQLDCBmcmFp -cjUwMEBmcmFpcjUwMC1QcmVjaXNpb24tNTUyMDEwMC4GA1UEAwwnbWtjZXJ0IGZy -YWlyNTAwQGZyYWlyNTAwLVByZWNpc2lvbi01NTIwMB4XDTE5MDEwOTIwMzQ1N1oX -DTI5MDEwOTIwMzQ1N1owVDEnMCUGA1UEChMebWtjZXJ0IGRldmVsb3BtZW50IGNl -cnRpZmljYXRlMSkwJwYDVQQLDCBmcmFpcjUwMEBmcmFpcjUwMC1QcmVjaXNpb24t -NTUyMDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALahGo80UFExe7Iv -jPDulPP9Vu3mPVW/4XhrvmbwjHPSXk6nvK34kdDmGsS/UVgtSMH+sdMNFavkhyK/ -b6PW5dPy+febfxlnaOkrZ5ptYx5IG1l/CNY/QDpQKGljW9YGQDV2t9apgKgT1/Ob -JIKf/rfd2o94iyxlrRnbXXidyMa1E6loo1AzzaN/g17dnblIL7ZCZtflgbsgnytw -UtwS92kTsvMHvuzM7Paz2M0xx+RNtQ2rq51fwph55gn7HLlBFEbkrMsfFj7hEquC -vJYvyrIEvaQLMyIOf+6/OgmrG9Z5ioMV4WAW9FLSuzXuuJruQc7FwQl4XIuE8d0M -jPjRfIcCAwEAAaOBizCBiDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB -BQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBTfMtd0Al3Ly09elEje6jyl -b3EQmjAyBgNVHREEKzApgglsb2NhbGhvc3SHBAAAAACHBH8AAAGHEAAAAAAAAAAA -AAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggGBADLu7RSMVnUiRNyTqIM3aMmkUXmL -xSPB/SZRifqVwmp9R6ygAZWzC7Lw5BpX2WCde1jqWJZw1AjYbe4w5i8e9jaiUyYZ -eaLuQN7/+dyWeMIfFKx7thDxmati+OkSJSoojROA1v4NY7QAIM6ycfFkwTBRokPz -42srfR+XXrvdNmBRqjpvpr48SAn44uvqAkVr3kNgqs1xycPgjsFvMO7qZlU6w/ev -/7QFUgtyZS/Saa4s3yRXHZ++g3SpPinrzf8VqmovL/MoaqB/tYVjOA/1B3QAkli6 -DIl+99eKANlqARXzMeXvgLpcg+1oAw0hYjFpCtqKhovhQzqN6KlAbmJ9JWTk35x8 -81nOERZH5dh6JZoHzaaB/ZMEjWkmHnyi4bf5dXiPLzfXJslbQKHhnSt4nfZiSodS -brUVv/sux119zyUPe9iA6NNPFS/No1XOKcHrG19jiXTq/HIdJRoIrN6eRJDTRVK1 -HyJ6uTvTJDu4ceBp2J1gz7R5opWbGyytDGg3Tw== ------END CERTIFICATE----- -""" -PRIVATE_KEY = b"""-----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC2oRqPNFBRMXuy -L4zw7pTz/Vbt5j1Vv+F4a75m8Ixz0l5Op7yt+JHQ5hrEv1FYLUjB/rHTDRWr5Ici -v2+j1uXT8vn3m38ZZ2jpK2eabWMeSBtZfwjWP0A6UChpY1vWBkA1drfWqYCoE9fz -mySCn/633dqPeIssZa0Z2114ncjGtROpaKNQM82jf4Ne3Z25SC+2QmbX5YG7IJ8r -cFLcEvdpE7LzB77szOz2s9jNMcfkTbUNq6udX8KYeeYJ+xy5QRRG5KzLHxY+4RKr -gryWL8qyBL2kCzMiDn/uvzoJqxvWeYqDFeFgFvRS0rs17ria7kHOxcEJeFyLhPHd -DIz40XyHAgMBAAECggEAZ1q7Liob/icz6r5wU/WhhIduB8qSEZI65qyLH7Sot+9p -Abh51jbjRsbChXAEeBOAppEeT+OKzTHSrH6MjrtSa+WJQ3DTuCvGupae1k1rl7qV -B8wV0zIOhjHQ/PuHAJOfCOK73ZclwXkhcLLvMaGcRLAgPaupj6GnGggEWPtqodDo -qBOcixT3/lMW5M1GklkqJqbD8g8qcx7SFBwORJjpwVX84Ynnursu0ZvTfK/CzZTk -D5t/UXyRV5Y5QBkzKIKzC0qUHv4eMIqkzlPBYx2PnAgrHokOm9/RS28yKT2DVPhw -t311ZM6+Z5AxfKamARWZbZdC8RG5Qo0ujLmgogNn2QKBgQDsqpwO+/yJlvF81nf9 -0Ye5o0OdOdD5q1ra46PyhQ56hIC5cRZx3s3E9hUFDcot81qj9nMTpSGJL5J6GqAY -W7p3PbpYxT27MDjthgHHcZy7hu1M9no65ZAK1ElxVhKMgl89RQu/HQoa6Uh3qjbF -X0edTBTBJoGOYQ1lVaoL8s307QKBgQDFjGtEKubolZ0OqFb361fDcYs0RDKNlNxy -RIMM6Dhl0tgGHxNFuFNlLdjKyPEltfNaK0L0W3i3Ndf5sUlr2MuXYgO6RRqWo/D2 -Tr2/jd6gsVKLK871WD7IS5SbCirCwuEsZQsZ2J2TWECoPqc8L3iZwyW6VGRkIW+K -o2Sl7P4cwwKBgQCnhAt6P7p82S6NInFEY28iYwGU5DuavUNN9BszqiKZbfh/SiCM -8RvM8jHmpeAZrkrWC7dgjF20cMvJSddP5n2RsUuZUeNj/7oLxfK0bSJ3SgXlmADk -d2EBiUmCw13VvuISyDCMUc25Rq5YpU6nXc2e9R8rqEnDscZ9l6kJVA+b8QKBgBAZ -coB6spjP4J3aMERCJMPj1AFtcWVCdXjGhpudrUL3HO3ayHpNHFbJlrpoB+cX3f5C -OlGpxru/optRzHcCkw0CSuV6TkFqmO+p2SLsT/Fuohh/eH1cNLmkFzdPa861jR5O -GcqAcc8ZSSOs/3oTMFPvqHp3+DqE0w9MY552Ivt7AoGATtJkMAg9M4U/5qIsCbRz -LplSCRvcarrg+czXW1re6y117rVjRHPCHgT//azsBDER0WpWSGv7XEnZwnz8U6Cn -FCXoiqqEJuD2wLwQlhb7QVXYTMdCwfPj5WV7ARJO1N4ty3g8x+jnTQCVoMpdhgxC -Sflxx+6bI4XMh0AsZhgtdW4= ------END PRIVATE KEY----- -""" +@pytest.fixture +def tls_certificate_authority() -> trustme.CA: + return trustme.CA() -@pytest.fixture(scope="function") -def certfile_and_keyfile(tmp_path): - certfile = str(tmp_path / "cert.pem") - with open(certfile, "bw") as fout: - fout.write(CERTIFICATE) +@pytest.fixture +def tls_certificate(tls_certificate_authority): + return tls_certificate_authority.issue_server_cert( + "localhost", + "127.0.0.1", + "::1", + ) - keyfile = str(tmp_path / "key.pem") - with open(keyfile, "bw") as fout: - fout.write(PRIVATE_KEY) - return certfile, keyfile +@pytest.fixture +def tls_ca_certificate_pem_path(tls_certificate_authority): + with tls_certificate_authority.cert_pem.tempfile() as ca_cert_pem: + yield ca_cert_pem + + +@pytest.fixture +def tls_ca_certificate_private_key_path(tls_certificate_authority): + with tls_certificate_authority.private_key_pem.tempfile() as private_key: + yield private_key + + +@pytest.fixture +def tls_certificate_pem_path(tls_certificate): + with tls_certificate.private_key_and_cert_chain_pem.tempfile() as cert_pem: + yield cert_pem diff --git a/tests/supervisors/test_watchgodreload.py b/tests/supervisors/test_watchgodreload.py index f8bc774fd..36a6e3261 100644 --- a/tests/supervisors/test_watchgodreload.py +++ b/tests/supervisors/test_watchgodreload.py @@ -11,7 +11,9 @@ def run(sockets): pass -def test_watchgodreload(certfile_and_keyfile): +def test_watchgodreload( + tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path +): config = Config(app=None) reloader = WatchGodReload(config, target=run, sockets=[]) reloader.signal_handler(sig=signal.SIGINT, frame=None) diff --git a/tests/test_config.py b/tests/test_config.py index d8052dc2e..bd1c26873 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -84,9 +84,22 @@ def test_socket_bind(): assert isinstance(config.bind_socket(), socket.socket) -def test_ssl_config(certfile_and_keyfile): - certfile, keyfile = certfile_and_keyfile - config = Config(app=asgi_app, ssl_certfile=certfile, ssl_keyfile=keyfile) +def test_ssl_config(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): + config = Config( + app=asgi_app, + ssl_certfile=tls_ca_certificate_pem_path, + ssl_keyfile=tls_ca_certificate_private_key_path, + ) + config.load() + + assert config.is_ssl is True + + +def test_ssl_config_combined(tls_certificate_pem_path): + config = Config( + app=asgi_app, + ssl_certfile=tls_certificate_pem_path, + ) config.load() assert config.is_ssl is True diff --git a/tests/test_ssl.py b/tests/test_ssl.py index b83d7dde3..e6b49367d 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -28,9 +28,42 @@ def no_ssl_verification(session=requests.Session): @pytest.mark.skipif( sys.platform.startswith("win"), reason="Skipping SSL test on Windows" ) -def test_run(certfile_and_keyfile): - certfile, keyfile = certfile_and_keyfile +def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): + class App: + def __init__(self, scope): + if scope["type"] != "http": + raise Exception() + + async def __call__(self, receive, send): + await send({"type": "http.response.start", "status": 204, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + class CustomServer(Server): + def install_signal_handlers(self): + pass + + config = Config( + app=App, + loop="asyncio", + limit_max_requests=1, + ssl_keyfile=tls_ca_certificate_private_key_path, + ssl_certfile=tls_ca_certificate_pem_path, + ) + server = CustomServer(config=config) + thread = threading.Thread(target=server.run) + thread.start() + while not server.started: + time.sleep(0.01) + with no_ssl_verification(): + response = requests.get("https://127.0.0.1:8000") + assert response.status_code == 204 + thread.join() + + +@pytest.mark.skipif( + sys.platform.startswith("win"), reason="Skipping SSL test on Windows" +) +def test_run_chain(tls_certificate_pem_path): class App: def __init__(self, scope): if scope["type"] != "http": @@ -48,8 +81,7 @@ def install_signal_handlers(self): app=App, loop="asyncio", limit_max_requests=1, - ssl_keyfile=keyfile, - ssl_certfile=certfile, + ssl_certfile=tls_certificate_pem_path, ) server = CustomServer(config=config) thread = threading.Thread(target=server.run) From 90dbb6e09bbca488edbc9c50fef63cd3067b7cdc Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 12 Oct 2020 12:54:34 +0200 Subject: [PATCH 057/128] Adding ability to decrypt ssl key file (#808) * adding parameter --ssl-password to decrypt ssl key file * rever changed to see if build passes * put back the changes after applying black * fixing E731 do not assign a lambda expression, use a def * fix formatting * !st pass using trustme * 2nd pass with more precise fixtures * 3rd pass * Forgot dev requirements.txt * test updated + black * Black * Added test to run uvicorn with a chain certificate ! * Added config test as well when using combined certificate and pk * Added fixture for encrypted key * Added test for passing the password * Switch naming * Docs upadted post name change * Updated for gunicorn worker, we should definitely test those! * Fucked up the merge... Typo * Added explicitely test reqs Co-authored-by: remster85 --- docs/deployment.md | 1 + docs/index.md | 1 + requirements.txt | 1 + tests/conftest.py | 18 ++++++++++++++++++ tests/test_ssl.py | 38 ++++++++++++++++++++++++++++++++++++++ uvicorn/config.py | 10 ++++++++-- uvicorn/main.py | 9 +++++++++ uvicorn/workers.py | 1 + 8 files changed, 77 insertions(+), 2 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 299e67564..053c4f7e6 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -79,6 +79,7 @@ Options: 5] --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file + --ssl-keyfile-password TEXT SSL key file password --ssl-version INTEGER SSL version to use (see stdlib ssl module's) [default: 2] --ssl-cert-reqs INTEGER Whether client certificate is required (see diff --git a/docs/index.md b/docs/index.md index 79f038b66..ae89e0454 100644 --- a/docs/index.md +++ b/docs/index.md @@ -148,6 +148,7 @@ Options: 5] --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file + --ssl-keyfile-password TEXT SSL key file password --ssl-version INTEGER SSL version to use (see stdlib ssl module's) [default: 2] --ssl-cert-reqs INTEGER Whether client certificate is required (see diff --git a/requirements.txt b/requirements.txt index fd61c1dee..7fde8b844 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ pytest-mock requests mypy trustme +cryptography coverage # Documentation diff --git a/tests/conftest.py b/tests/conftest.py index f396cb415..00428b982 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,7 @@ import pytest import trustme +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization @pytest.fixture @@ -28,6 +30,22 @@ def tls_ca_certificate_private_key_path(tls_certificate_authority): yield private_key +@pytest.fixture +def tls_ca_certificate_private_key_encrypted_path(tls_certificate_authority): + private_key = serialization.load_pem_private_key( + tls_certificate_authority.private_key_pem.bytes(), + password=None, + backend=default_backend(), + ) + encrypted_key = private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.BestAvailableEncryption(b"uvicorn password for the win"), + ) + with trustme.Blob(encrypted_key).tempfile() as private_encrypted_key: + yield private_encrypted_key + + @pytest.fixture def tls_certificate_pem_path(tls_certificate): with tls_certificate.private_key_and_cert_chain_pem.tempfile() as cert_pem: diff --git a/tests/test_ssl.py b/tests/test_ssl.py index e6b49367d..d64d828ab 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -92,3 +92,41 @@ def install_signal_handlers(self): response = requests.get("https://127.0.0.1:8000") assert response.status_code == 204 thread.join() + + +@pytest.mark.skipif( + sys.platform.startswith("win"), reason="Skipping SSL test on Windows" +) +def test_run_password( + tls_ca_certificate_pem_path, tls_ca_certificate_private_key_encrypted_path +): + class App: + def __init__(self, scope): + if scope["type"] != "http": + raise Exception() + + async def __call__(self, receive, send): + await send({"type": "http.response.start", "status": 204, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + class CustomServer(Server): + def install_signal_handlers(self): + pass + + config = Config( + app=App, + loop="asyncio", + limit_max_requests=1, + ssl_keyfile=tls_ca_certificate_private_key_encrypted_path, + ssl_certfile=tls_ca_certificate_pem_path, + ssl_keyfile_password="uvicorn password for the win", + ) + server = CustomServer(config=config) + thread = threading.Thread(target=server.run) + thread.start() + while not server.started: + time.sleep(0.01) + with no_ssl_verification(): + response = requests.get("https://127.0.0.1:8000") + assert response.status_code == 204 + thread.join() diff --git a/uvicorn/config.py b/uvicorn/config.py index 944317e6c..c5be3564a 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -101,9 +101,12 @@ logger = logging.getLogger("uvicorn.error") -def create_ssl_context(certfile, keyfile, ssl_version, cert_reqs, ca_certs, ciphers): +def create_ssl_context( + certfile, keyfile, password, ssl_version, cert_reqs, ca_certs, ciphers +): ctx = ssl.SSLContext(ssl_version) - ctx.load_cert_chain(certfile, keyfile) + get_password = (lambda: password) if password else None + ctx.load_cert_chain(certfile, keyfile, get_password) ctx.verify_mode = cert_reqs if ca_certs: ctx.load_verify_locations(ca_certs) @@ -146,6 +149,7 @@ def __init__( callback_notify=None, ssl_keyfile=None, ssl_certfile=None, + ssl_keyfile_password=None, ssl_version=SSL_PROTOCOL_VERSION, ssl_cert_reqs=ssl.CERT_NONE, ssl_ca_certs=None, @@ -180,6 +184,7 @@ def __init__( self.callback_notify = callback_notify self.ssl_keyfile = ssl_keyfile self.ssl_certfile = ssl_certfile + self.ssl_keyfile_password = ssl_keyfile_password self.ssl_version = ssl_version self.ssl_cert_reqs = ssl_cert_reqs self.ssl_ca_certs = ssl_ca_certs @@ -266,6 +271,7 @@ def load(self): self.ssl = create_ssl_context( keyfile=self.ssl_keyfile, certfile=self.ssl_certfile, + password=self.ssl_keyfile_password, ssl_version=self.ssl_version, cert_reqs=self.ssl_cert_reqs, ca_certs=self.ssl_ca_certs, diff --git a/uvicorn/main.py b/uvicorn/main.py index e243ebb03..feec87383 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -227,6 +227,13 @@ def print_version(ctx, param, value): help="SSL certificate file", show_default=True, ) +@click.option( + "--ssl-keyfile-password", + type=str, + default=None, + help="SSL keyfile password", + show_default=True, +) @click.option( "--ssl-version", type=int, @@ -306,6 +313,7 @@ def main( timeout_keep_alive: int, ssl_keyfile: str, ssl_certfile: str, + ssl_keyfile_password: str, ssl_version: int, ssl_cert_reqs: int, ssl_ca_certs: str, @@ -345,6 +353,7 @@ def main( "timeout_keep_alive": timeout_keep_alive, "ssl_keyfile": ssl_keyfile, "ssl_certfile": ssl_certfile, + "ssl_keyfile_password": ssl_keyfile_password, "ssl_version": ssl_version, "ssl_cert_reqs": ssl_cert_reqs, "ssl_ca_certs": ssl_ca_certs, diff --git a/uvicorn/workers.py b/uvicorn/workers.py index 57f86c7b2..1ac036372 100644 --- a/uvicorn/workers.py +++ b/uvicorn/workers.py @@ -42,6 +42,7 @@ def __init__(self, *args, **kwargs): ssl_kwargs = { "ssl_keyfile": self.cfg.ssl_options.get("keyfile"), "ssl_certfile": self.cfg.ssl_options.get("certfile"), + "ssl_keyfile_password": self.cfg.ssl_options.get("password"), "ssl_version": self.cfg.ssl_options.get("ssl_version"), "ssl_cert_reqs": self.cfg.ssl_options.get("cert_reqs"), "ssl_ca_certs": self.cfg.ssl_options.get("ca_certs"), From 5acaee5b9ef477c8210b0d7b5a9c88ad11d217d0 Mon Sep 17 00:00:00 2001 From: euri10 Date: Wed, 14 Oct 2020 10:26:12 +0200 Subject: [PATCH 058/128] Fix reload with ipv6 host (#803) * Handling of ipv6 host Totally unrelated but useful commit to trigger travis Ignore all coverage generated files * Should not be part of PR * Should not be part of PR * Should not be part of PR * Added enums Make message private * Identity comparison --- uvicorn/config.py | 37 ++++++++++++++++++++++++++++++------- uvicorn/main.py | 14 ++++++++------ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/uvicorn/config.py b/uvicorn/config.py index c5be3564a..a36b89f8d 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -7,6 +7,7 @@ import socket import ssl import sys +from enum import Enum from typing import List, Tuple import click @@ -115,6 +116,27 @@ def create_ssl_context( return ctx +class _IPKind(Enum): + IPv4 = "IPv4" + IPv6 = "IPv6" + + +def _get_server_start_message( + host_ip_version: _IPKind = _IPKind.IPv4, +) -> Tuple[str, str]: + if host_ip_version is _IPKind.IPv6: + ip_repr = "%s://[%s]:%d" + else: + ip_repr = "%s://%s:%d" + message = f"Uvicorn running on {ip_repr} (Press CTRL+C to quit)" + color_message = ( + "Uvicorn running on " + + click.style(ip_repr, bold=True) + + " (Press CTRL+C to quit)" + ) + return message, color_message + + class Config: def __init__( self, @@ -341,7 +363,10 @@ def setup_event_loop(self): loop_setup() def bind_socket(self): - sock = socket.socket() + family, sockettype, proto, canonname, sockaddr = socket.getaddrinfo( + self.host, self.port, type=socket.SOCK_STREAM + )[0] + sock = socket.socket(family=family, type=sockettype) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind((self.host, self.port)) @@ -350,12 +375,10 @@ def bind_socket(self): sys.exit(1) sock.set_inheritable(True) - message = "Uvicorn running on %s://%s:%d (Press CTRL+C to quit)" - color_message = ( - "Uvicorn running on " - + click.style("%s://%s:%d", bold=True) - + " (Press CTRL+C to quit)" - ) + if family == socket.AddressFamily.AF_INET6: + message, color_message = _get_server_start_message(_IPKind.IPv6) + else: + message, color_message = _get_server_start_message(_IPKind.IPv4) protocol_name = "https" if self.is_ssl else "http" logger.info( message, diff --git a/uvicorn/main.py b/uvicorn/main.py index feec87383..fe6547ab5 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -10,6 +10,7 @@ import time import typing from email.utils import formatdate +from ipaddress import IPv6Address, ip_address import click @@ -24,6 +25,8 @@ SSL_PROTOCOL_VERSION, WS_PROTOCOLS, Config, + _get_server_start_message, + _IPKind, ) from uvicorn.supervisors import ChangeReload, Multiprocess @@ -520,12 +523,11 @@ def _share_socket(sock: socket) -> socket: if port == 0: port = server.sockets[0].getsockname()[1] protocol_name = "https" if config.ssl else "http" - message = "Uvicorn running on %s://%s:%d (Press CTRL+C to quit)" - color_message = ( - "Uvicorn running on " - + click.style("%s://%s:%d", bold=True) - + " (Press CTRL+C to quit)" - ) + if isinstance(ip_address(config.host), IPv6Address): + message, color_message = _get_server_start_message(_IPKind.IPv6) + else: + message, color_message = _get_server_start_message(_IPKind.IPv4) + logger.info( message, protocol_name, From 1b32f9971ad4080c67e5779d519dec6b60d3ece6 Mon Sep 17 00:00:00 2001 From: Lucas Date: Wed, 14 Oct 2020 15:12:59 +0200 Subject: [PATCH 059/128] Fixes watchgod with common prefixes (#817) * Fixes watchgod with common prefixes Use pathlib to recognize common ancestry instead of string compare * Add tests for watchgodreload Test for common prefix paths Replace os.path with pathlib --- tests/supervisors/test_watchgodreload.py | 43 ++++++++++++++++++++++-- uvicorn/supervisors/watchgodreload.py | 11 +++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/supervisors/test_watchgodreload.py b/tests/supervisors/test_watchgodreload.py index 36a6e3261..4264a452d 100644 --- a/tests/supervisors/test_watchgodreload.py +++ b/tests/supervisors/test_watchgodreload.py @@ -22,7 +22,7 @@ def test_watchgodreload( def test_should_reload_when_python_file_is_changed(tmpdir): file = "example.py" - update_file = Path(os.path.join(str(tmpdir), file)) + update_file = Path(tmpdir).joinpath(file) update_file.touch() working_dir = os.getcwd() @@ -46,7 +46,7 @@ def test_should_reload_when_python_file_is_changed(tmpdir): def test_should_not_reload_when_dot_file_is_changed(tmpdir): file = ".dotted" - update_file = Path(os.path.join(str(tmpdir), file)) + update_file = Path(tmpdir).joinpath(file) update_file.touch() working_dir = os.getcwd() @@ -66,3 +66,42 @@ def test_should_not_reload_when_dot_file_is_changed(tmpdir): reloader.shutdown() finally: os.chdir(working_dir) + + +def test_should_reload_when_directories_have_same_prefix(tmpdir): + file = "example.py" + tmpdir_path = Path(tmpdir) + app_dir = tmpdir_path.joinpath("app") + app_ext_dir = tmpdir_path.joinpath("app_extension") + app_file = app_dir.joinpath(file) + app_ext_file = app_ext_dir.joinpath(file) + app_dir.mkdir() + app_ext_dir.mkdir() + app_file.touch() + app_ext_file.touch() + + working_dir = os.getcwd() + os.chdir(str(tmpdir)) + try: + config = Config( + app=None, reload=True, reload_dirs=[str(app_dir), str(app_ext_dir)] + ) + reloader = WatchGodReload(config, target=run, sockets=[]) + reloader.signal_handler(sig=signal.SIGINT, frame=None) + reloader.startup() + + assert not reloader.should_restart() + time.sleep(0.1) + app_file.touch() + assert reloader.should_restart() + + reloader.restart() + + assert not reloader.should_restart() + time.sleep(0.1) + app_ext_file.touch() + assert reloader.should_restart() + + reloader.shutdown() + finally: + os.chdir(working_dir) diff --git a/uvicorn/supervisors/watchgodreload.py b/uvicorn/supervisors/watchgodreload.py index 9393aedb0..d6af23d23 100644 --- a/uvicorn/supervisors/watchgodreload.py +++ b/uvicorn/supervisors/watchgodreload.py @@ -1,6 +1,6 @@ import logging import re -from os import path +from pathlib import Path from watchgod import DefaultWatcher @@ -30,9 +30,9 @@ def __init__(self, config, target, sockets): self.reloader_name = "watchgod" self.watchers = [] watch_dirs = { - path.realpath(watch_dir) + Path(watch_dir).resolve() for watch_dir in self.config.reload_dirs - if path.isdir(watch_dir) + if Path(watch_dir).is_dir() } watch_dirs_set = set(watch_dirs) @@ -43,10 +43,9 @@ def __init__(self, config, target, sockets): if compare_dir is watch_dir: continue - if watch_dir.startswith(compare_dir) and len(watch_dir) > len( - compare_dir - ): + if compare_dir in watch_dir.parents: watch_dirs_set.remove(watch_dir) + self.watch_dir_set = watch_dirs_set for w in watch_dirs_set: self.watchers.append(CustomWatcher(w)) From c592c7b8b7af53ba5ba482d0daddf5e9fc2e8cf1 Mon Sep 17 00:00:00 2001 From: cdeler Date: Mon, 19 Oct 2020 10:16:38 +0300 Subject: [PATCH 060/128] Changed test_config fixture scope, which fixes some config tests (#821) * Changed the tests order, which breaks the test suite * Changed the test fixture scope, which fixes the test suite --- tests/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index bd1c26873..7428b31d9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,7 +17,7 @@ def mocked_logging_config_module(mocker): return mocker.patch("logging.config") -@pytest.fixture +@pytest.fixture(scope="function") def logging_config(): return deepcopy(LOGGING_CONFIG) @@ -125,9 +125,9 @@ def test_asgi_version(app, expected_interface): "use_colors, expected", [ pytest.param(None, None, id="use_colors_not_provided"), + pytest.param("invalid", None, id="use_colors_invalid_value"), pytest.param(True, True, id="use_colors_enabled"), pytest.param(False, False, id="use_colors_disabled"), - pytest.param("invalid", False, id="use_colors_invalid_value"), ], ) def test_log_config_default(mocked_logging_config_module, use_colors, expected): From cca3c3e1968937bbd0cb366dcefc8b33dbac39b9 Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 19 Oct 2020 17:45:56 +0200 Subject: [PATCH 061/128] v0.12.2 (#823) --- CHANGELOG.md | 14 ++++++++++++++ uvicorn/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7e79f3d..8367d22bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Change Log +## 0.12.2 - 2020-10-19 + +### Added +- Adding ability to decrypt ssl key file (#808) 10/12/20 90dbb6e0 +- Support .yml log config files (#799) 10/6/20 b468950e +- Added python 3.9 support (#804) 10/6/20 08fd0559 + +### Fixed +- Fixes watchgod with common prefixes (#817) 10/14/20 1b32f997 +- Fix reload with ipv6 host (#803) 10/14/20 5acaee5b +- Added cli suport for headers containing colon (#813) 10/12/20 68732899 +- Sharing socket across workers on windows (#802) 10/12/20 103167a0 +- Note the need to configure trusted "ips" when using unix sockets (#796) 10/4/20 a504c569 + ## 0.12.1 - 2020-09-30 ### Changed diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 7d0702b3b..2e4d85ce1 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.12.1" +__version__ = "0.12.2" __all__ = ["main", "run", "Config", "Server"] From d5614f628eedbdf94953f5ebd36554168036b339 Mon Sep 17 00:00:00 2001 From: Vibhu Agarwal Date: Wed, 21 Oct 2020 16:05:46 +0530 Subject: [PATCH 062/128] [Docs] Fix mkdocs warning + Added Permalinks (#826) * [Docs] Added toc:permalink markdown_extension * [Docs] Fix mkdocs warning: 'pages'->'nav' --- mkdocs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 0bd324317..a37e1da8a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,7 +8,7 @@ repo_name: encode/uvicorn repo_url: https://github.com/encode/uvicorn edit_uri: "" -pages: +nav: - Introduction: 'index.md' - Settings: 'settings.md' - Deployment: 'deployment.md' @@ -17,6 +17,8 @@ pages: markdown_extensions: - markdown.extensions.codehilite: guess_lang: false + - toc: + permalink: true extra_javascript: - 'js/chat.js' From d78394e31310bb60defca35f0c6b1104820cf687 Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 22 Oct 2020 09:25:03 +0200 Subject: [PATCH 063/128] Fix server not running with explicit hostname (#827) * Fix host * Change signature of message function * Lint --- tests/test_main.py | 25 +++++++++++++++++++++++++ uvicorn/config.py | 16 ++++------------ uvicorn/main.py | 17 +++++++++++------ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 8b31393e7..1f9241a26 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -33,6 +33,31 @@ def install_signal_handlers(self): thread.join() +def test_run_hostname(): + class App: + def __init__(self, scope): + if scope["type"] != "http": + raise Exception() + + async def __call__(self, receive, send): + await send({"type": "http.response.start", "status": 204, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + class CustomServer(Server): + def install_signal_handlers(self): + pass + + config = Config(app=App, host="localhost", loop="asyncio", limit_max_requests=1) + server = CustomServer(config=config) + thread = threading.Thread(target=server.run) + thread.start() + while not server.started: + time.sleep(0.01) + response = requests.get("http://localhost:8000") + assert response.status_code == 204 + thread.join() + + def test_run_multiprocess(): class App: def __init__(self, scope): diff --git a/uvicorn/config.py b/uvicorn/config.py index a36b89f8d..c48dfc173 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -7,7 +7,6 @@ import socket import ssl import sys -from enum import Enum from typing import List, Tuple import click @@ -116,15 +115,8 @@ def create_ssl_context( return ctx -class _IPKind(Enum): - IPv4 = "IPv4" - IPv6 = "IPv6" - - -def _get_server_start_message( - host_ip_version: _IPKind = _IPKind.IPv4, -) -> Tuple[str, str]: - if host_ip_version is _IPKind.IPv6: +def _get_server_start_message(is_ipv6_message: bool = False) -> Tuple[str, str]: + if is_ipv6_message: ip_repr = "%s://[%s]:%d" else: ip_repr = "%s://%s:%d" @@ -376,9 +368,9 @@ def bind_socket(self): sock.set_inheritable(True) if family == socket.AddressFamily.AF_INET6: - message, color_message = _get_server_start_message(_IPKind.IPv6) + message, color_message = _get_server_start_message(is_ipv6_message=True) else: - message, color_message = _get_server_start_message(_IPKind.IPv4) + message, color_message = _get_server_start_message() protocol_name = "https" if self.is_ssl else "http" logger.info( message, diff --git a/uvicorn/main.py b/uvicorn/main.py index fe6547ab5..da89ad184 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -10,7 +10,7 @@ import time import typing from email.utils import formatdate -from ipaddress import IPv6Address, ip_address +from ipaddress import IPv4Address, IPv6Address, ip_address import click @@ -26,7 +26,6 @@ WS_PROTOCOLS, Config, _get_server_start_message, - _IPKind, ) from uvicorn.supervisors import ChangeReload, Multiprocess @@ -523,10 +522,16 @@ def _share_socket(sock: socket) -> socket: if port == 0: port = server.sockets[0].getsockname()[1] protocol_name = "https" if config.ssl else "http" - if isinstance(ip_address(config.host), IPv6Address): - message, color_message = _get_server_start_message(_IPKind.IPv6) - else: - message, color_message = _get_server_start_message(_IPKind.IPv4) + try: + addr = ip_address(config.host) + if isinstance(addr, IPv6Address): + message, color_message = _get_server_start_message( + is_ipv6_message=True + ) + elif isinstance(addr, IPv4Address): + message, color_message = _get_server_start_message() + except ValueError: + message, color_message = _get_server_start_message() logger.info( message, From ef806259fa1d27b766d7c8357184d6f059a261d4 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 26 Oct 2020 12:56:15 +0100 Subject: [PATCH 064/128] Add reload test base class (#833) * Add reload test base class * Use custom test class for reloader setup/teardown * Use tmpdir.as_cwd from pytest to switch context * Remove unused run function * Refactor more * More precise message Co-authored-by: euri10 --- tests/supervisors/test_reload.py | 98 +++++++++++++++++++++ tests/supervisors/test_statreload.py | 47 ---------- tests/supervisors/test_watchgodreload.py | 107 ----------------------- uvicorn/supervisors/statreload.py | 2 +- uvicorn/supervisors/watchgodreload.py | 2 +- 5 files changed, 100 insertions(+), 156 deletions(-) create mode 100644 tests/supervisors/test_reload.py delete mode 100644 tests/supervisors/test_statreload.py delete mode 100644 tests/supervisors/test_watchgodreload.py diff --git a/tests/supervisors/test_reload.py b/tests/supervisors/test_reload.py new file mode 100644 index 000000000..b08eed406 --- /dev/null +++ b/tests/supervisors/test_reload.py @@ -0,0 +1,98 @@ +import signal +from pathlib import Path +from time import sleep + +import pytest + +from uvicorn.config import Config +from uvicorn.supervisors.basereload import BaseReload +from uvicorn.supervisors.statreload import StatReload +from uvicorn.supervisors.watchgodreload import WatchGodReload + + +@pytest.mark.parametrize("reloader_class", [StatReload, WatchGodReload]) +class TestBaseReload: + tmp_path: Path + + @pytest.fixture(autouse=True) + def setup(self, tmpdir, reloader_class): + self.tmpdir = tmpdir + self.tmp_path = Path(tmpdir) + self.reloader_class = reloader_class + + def run(self, sockets): + pass + + def _setup_reloader(self, config: Config) -> BaseReload: + reloader = self.reloader_class(config, target=self.run, sockets=[]) + reloader.signal_handler(sig=signal.SIGINT, frame=None) + reloader.startup() + return reloader + + def _reload_tester(self, reloader: BaseReload, file: Path) -> bool: + reloader.restart() + assert not reloader.should_restart() + sleep(0.1) + file.touch() + return reloader.should_restart() + + def test_reloader_should_initialize(self): + """ + A basic sanity check. + + Simply run the reloader against a no-op server, and signal for it to + quit immediately. + """ + config = Config(app=None, reload=True) + reloader = self._setup_reloader(config) + reloader.shutdown() + + def test_should_reload_when_python_file_is_changed(self): + file = "example.py" + update_file = self.tmp_path.joinpath(file) + update_file.touch() + + with self.tmpdir.as_cwd(): + config = Config(app=None, reload=True) + reloader = self._setup_reloader(config) + + assert self._reload_tester(reloader, update_file) + + reloader.shutdown() + + def test_should_not_reload_when_dot_file_is_changed(self): + file = ".dotted" + + update_file = self.tmp_path.joinpath(file) + update_file.touch() + + with self.tmpdir.as_cwd(): + config = Config(app=None, reload=True) + reloader = self._setup_reloader(config) + + assert not self._reload_tester(reloader, update_file) + + reloader.shutdown() + + def test_should_reload_when_directories_have_same_prefix(self): + file = "example.py" + + app_dir = self.tmp_path.joinpath("app") + app_ext_dir = self.tmp_path.joinpath("app_extension") + app_file = app_dir.joinpath(file) + app_ext_file = app_ext_dir.joinpath(file) + app_dir.mkdir() + app_ext_dir.mkdir() + app_file.touch() + app_ext_file.touch() + + with self.tmpdir.as_cwd(): + config = Config( + app=None, reload=True, reload_dirs=[str(app_dir), str(app_ext_dir)] + ) + reloader = self._setup_reloader(config) + + assert self._reload_tester(reloader, app_file) + assert self._reload_tester(reloader, app_ext_file) + + reloader.shutdown() diff --git a/tests/supervisors/test_statreload.py b/tests/supervisors/test_statreload.py deleted file mode 100644 index 891a4fce8..000000000 --- a/tests/supervisors/test_statreload.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import signal -import time -from pathlib import Path - -from uvicorn.config import Config -from uvicorn.supervisors.statreload import StatReload - - -def run(sockets): - pass - - -def test_statreload(): - """ - A basic sanity check. - - Simply run the reloader against a no-op server, and signal for it to - quit immediately. - """ - config = Config(app=None, reload=True) - reloader = StatReload(config, target=run, sockets=[]) - reloader.signal_handler(sig=signal.SIGINT, frame=None) - reloader.run() - - -def test_should_reload(tmpdir): - update_file = Path(os.path.join(str(tmpdir), "example.py")) - update_file.touch() - - working_dir = os.getcwd() - os.chdir(str(tmpdir)) - try: - config = Config(app=None, reload=True) - reloader = StatReload(config, target=run, sockets=[]) - reloader.signal_handler(sig=signal.SIGINT, frame=None) - reloader.startup() - - assert not reloader.should_restart() - time.sleep(0.1) - update_file.touch() - assert reloader.should_restart() - - reloader.restart() - reloader.shutdown() - finally: - os.chdir(working_dir) diff --git a/tests/supervisors/test_watchgodreload.py b/tests/supervisors/test_watchgodreload.py deleted file mode 100644 index 4264a452d..000000000 --- a/tests/supervisors/test_watchgodreload.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -import signal -import time -from pathlib import Path - -from uvicorn.config import Config -from uvicorn.supervisors.watchgodreload import WatchGodReload - - -def run(sockets): - pass - - -def test_watchgodreload( - tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path -): - config = Config(app=None) - reloader = WatchGodReload(config, target=run, sockets=[]) - reloader.signal_handler(sig=signal.SIGINT, frame=None) - reloader.run() - - -def test_should_reload_when_python_file_is_changed(tmpdir): - file = "example.py" - update_file = Path(tmpdir).joinpath(file) - update_file.touch() - - working_dir = os.getcwd() - os.chdir(str(tmpdir)) - try: - config = Config(app=None, reload=True) - reloader = WatchGodReload(config, target=run, sockets=[]) - reloader.signal_handler(sig=signal.SIGINT, frame=None) - reloader.startup() - - assert not reloader.should_restart() - time.sleep(0.1) - update_file.touch() - assert reloader.should_restart() - - reloader.restart() - reloader.shutdown() - finally: - os.chdir(working_dir) - - -def test_should_not_reload_when_dot_file_is_changed(tmpdir): - file = ".dotted" - update_file = Path(tmpdir).joinpath(file) - update_file.touch() - - working_dir = os.getcwd() - os.chdir(str(tmpdir)) - try: - config = Config(app=None, reload=True) - reloader = WatchGodReload(config, target=run, sockets=[]) - reloader.signal_handler(sig=signal.SIGINT, frame=None) - reloader.startup() - - assert not reloader.should_restart() - time.sleep(0.1) - update_file.touch() - assert not reloader.should_restart() - - reloader.restart() - reloader.shutdown() - finally: - os.chdir(working_dir) - - -def test_should_reload_when_directories_have_same_prefix(tmpdir): - file = "example.py" - tmpdir_path = Path(tmpdir) - app_dir = tmpdir_path.joinpath("app") - app_ext_dir = tmpdir_path.joinpath("app_extension") - app_file = app_dir.joinpath(file) - app_ext_file = app_ext_dir.joinpath(file) - app_dir.mkdir() - app_ext_dir.mkdir() - app_file.touch() - app_ext_file.touch() - - working_dir = os.getcwd() - os.chdir(str(tmpdir)) - try: - config = Config( - app=None, reload=True, reload_dirs=[str(app_dir), str(app_ext_dir)] - ) - reloader = WatchGodReload(config, target=run, sockets=[]) - reloader.signal_handler(sig=signal.SIGINT, frame=None) - reloader.startup() - - assert not reloader.should_restart() - time.sleep(0.1) - app_file.touch() - assert reloader.should_restart() - - reloader.restart() - - assert not reloader.should_restart() - time.sleep(0.1) - app_ext_file.touch() - assert reloader.should_restart() - - reloader.shutdown() - finally: - os.chdir(working_dir) diff --git a/uvicorn/supervisors/statreload.py b/uvicorn/supervisors/statreload.py index 3209c2ced..eeb98ca46 100644 --- a/uvicorn/supervisors/statreload.py +++ b/uvicorn/supervisors/statreload.py @@ -28,7 +28,7 @@ def should_restart(self): display_path = os.path.normpath(filename) if Path.cwd() in Path(filename).parents: display_path = os.path.normpath(os.path.relpath(filename)) - message = "Detected file change in '%s'. Reloading..." + message = "StatReload detected file change in '%s'. Reloading..." logger.warning(message, display_path) return True return False diff --git a/uvicorn/supervisors/watchgodreload.py b/uvicorn/supervisors/watchgodreload.py index d6af23d23..47ca94431 100644 --- a/uvicorn/supervisors/watchgodreload.py +++ b/uvicorn/supervisors/watchgodreload.py @@ -54,7 +54,7 @@ def should_restart(self): for watcher in self.watchers: change = watcher.check() if change != set(): - message = "Detected file change in '%s'. Reloading..." + message = "WatchGodReload detected file change in '%s'. Reloading..." logger.warning(message, [c[1] for c in change]) return True From d5dcf80cf7b666ab0dbbb4299fd6e9a13f80ad6f Mon Sep 17 00:00:00 2001 From: cdeler Date: Mon, 26 Oct 2020 21:44:02 +0300 Subject: [PATCH 065/128] Cancel old keepalive-trigger before setting new one. (#832) --- uvicorn/protocols/http/h11_impl.py | 7 ++++++- uvicorn/protocols/http/httptools_impl.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index fb70038ef..6a362ff20 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -153,11 +153,14 @@ def connection_lost(self, exc): def eof_received(self): pass - def data_received(self, data): + def _unset_keepalive_if_required(self): if self.timeout_keep_alive_task is not None: self.timeout_keep_alive_task.cancel() self.timeout_keep_alive_task = None + def data_received(self, data): + self._unset_keepalive_if_required() + self.conn.receive_data(data) self.handle_events() @@ -299,6 +302,8 @@ def on_response_complete(self): return # Set a short Keep-Alive timeout. + self._unset_keepalive_if_required() + self.timeout_keep_alive_task = self.loop.call_later( self.timeout_keep_alive, self.timeout_keep_alive_handler ) diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index 1642b7abb..15262326b 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -153,11 +153,14 @@ def connection_lost(self, exc): def eof_received(self): pass - def data_received(self, data): + def _unset_keepalive_if_required(self): if self.timeout_keep_alive_task is not None: self.timeout_keep_alive_task.cancel() self.timeout_keep_alive_task = None + def data_received(self, data): + self._unset_keepalive_if_required() + try: self.parser.feed_data(data) except httptools.HttpParserError: @@ -301,6 +304,8 @@ def on_response_complete(self): return # Set a short Keep-Alive timeout. + self._unset_keepalive_if_required() + self.timeout_keep_alive_task = self.loop.call_later( self.timeout_keep_alive, self.timeout_keep_alive_handler ) From 6468b70c85e10ae2d405165ca69f2b6a5bd55878 Mon Sep 17 00:00:00 2001 From: remster85 Date: Mon, 2 Nov 2020 06:40:44 -0500 Subject: [PATCH 066/128] Documentation: HTTPS settings add missing --ssl-keyfile-password (#839) * adding parameter --ssl-password to decrypt ssl key file * rever changed to see if build passes * put back the changes after applying black * fixing E731 do not assign a lambda expression, use a def * fix formatting * amend ssl documentation adding ssl-keyfile-passwor * remove duplicated text * amend incomplete documentation about https --ssl-keyfile-password * fix the parameter https --ssl-keyfile-password type * refine the parameter --ssl-keyfile-password description --- docs/settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/settings.md b/docs/settings.md index 617ac63ef..5836186b9 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -63,6 +63,7 @@ connecting IPs in the `forwarded-allow-ips` configuration. ## HTTPS * `--ssl-keyfile ` - SSL key file +* `--ssl-keyfile-password ` - Password to decrypt the ssl key * `--ssl-certfile ` - SSL certificate file * `--ssl-version ` - SSL version to use (see stdlib ssl module's) * `--ssl-cert-reqs ` - Whether client certificate is required (see stdlib ssl module's) From bdab488ebfa84933d4b5f45c164366321d280015 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sun, 8 Nov 2020 11:20:01 +0100 Subject: [PATCH 067/128] Rework IPv6 support (#837) * Revert "Fix server not running with explicit hostname (#827)" This reverts commit d78394e31310bb60defca35f0c6b1104820cf687. * Revert "Fix reload with ipv6 host (#803)" This reverts commit 5acaee5b9ef477c8210b0d7b5a9c88ad11d217d0. * Rework IPv6 support * Fix IPv6 localhost equivalent Co-authored-by: euri10 * Reduce diff size * More diff size reduction * Fix: self.host -> config.host Co-authored-by: euri10 --- docs/settings.md | 2 +- tests/test_main.py | 40 ++++++++++++---------------------------- uvicorn/config.py | 37 +++++++++++++++---------------------- uvicorn/main.py | 24 +++++++++++------------- 4 files changed, 39 insertions(+), 64 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 5836186b9..dc5682e24 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -11,7 +11,7 @@ equivalent keyword arguments, eg. `uvicorn.run("example:app", port=5000, reload= ## Socket Binding -* `--host ` - Bind socket to this host. Use `--host 0.0.0.0` to make the application available on your local network. **Default:** *'127.0.0.1'*. +* `--host ` - Bind socket to this host. Use `--host 0.0.0.0` to make the application available on your local network. IPv6 addresses are supported, for example: `--host '::'`. **Default:** *'127.0.0.1'*. * `--port ` - Bind to a socket with this port. **Default:** *8000*. * `--uds ` - Bind to a UNIX domain socket. Useful if you want to run Uvicorn behind a reverse proxy. * `--fd ` - Bind to socket from this file descriptor. Useful if you want to run Uvicorn within a process manager. diff --git a/tests/test_main.py b/tests/test_main.py index 1f9241a26..a04178f14 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,13 +2,22 @@ import threading import time +import pytest import requests from uvicorn.config import Config from uvicorn.main import Server -def test_run(): +@pytest.mark.parametrize( + "host, url", + [ + pytest.param(None, "http://127.0.0.1:8000", id="default"), + pytest.param("localhost", "http://127.0.0.1:8000", id="hostname"), + pytest.param("::1", "http://[::1]:8000", id="ipv6"), + ], +) +def test_run(host, url): class App: def __init__(self, scope): if scope["type"] != "http": @@ -22,38 +31,13 @@ class CustomServer(Server): def install_signal_handlers(self): pass - config = Config(app=App, loop="asyncio", limit_max_requests=1) + config = Config(app=App, host=host, loop="asyncio", limit_max_requests=1) server = CustomServer(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") - assert response.status_code == 204 - thread.join() - - -def test_run_hostname(): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - - class CustomServer(Server): - def install_signal_handlers(self): - pass - - config = Config(app=App, host="localhost", loop="asyncio", limit_max_requests=1) - server = CustomServer(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://localhost:8000") + response = requests.get(url) assert response.status_code == 204 thread.join() diff --git a/uvicorn/config.py b/uvicorn/config.py index c48dfc173..0b7bd1ef0 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -115,20 +115,6 @@ def create_ssl_context( return ctx -def _get_server_start_message(is_ipv6_message: bool = False) -> Tuple[str, str]: - if is_ipv6_message: - ip_repr = "%s://[%s]:%d" - else: - ip_repr = "%s://%s:%d" - message = f"Uvicorn running on {ip_repr} (Press CTRL+C to quit)" - color_message = ( - "Uvicorn running on " - + click.style(ip_repr, bold=True) - + " (Press CTRL+C to quit)" - ) - return message, color_message - - class Config: def __init__( self, @@ -355,10 +341,15 @@ def setup_event_loop(self): loop_setup() def bind_socket(self): - family, sockettype, proto, canonname, sockaddr = socket.getaddrinfo( - self.host, self.port, type=socket.SOCK_STREAM - )[0] - sock = socket.socket(family=family, type=sockettype) + family = socket.AF_INET + addr_format = "%s://%s:%d" + + if self.host and ":" in self.host: + # It's an IPv6 address. + family = socket.AF_INET6 + addr_format = "%s://[%s]:%d" + + sock = socket.socket(family=family) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) try: sock.bind((self.host, self.port)) @@ -367,10 +358,12 @@ def bind_socket(self): sys.exit(1) sock.set_inheritable(True) - if family == socket.AddressFamily.AF_INET6: - message, color_message = _get_server_start_message(is_ipv6_message=True) - else: - message, color_message = _get_server_start_message() + message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" + color_message = ( + "Uvicorn running on " + + click.style(addr_format, bold=True) + + " (Press CTRL+C to quit)" + ) protocol_name = "https" if self.is_ssl else "http" logger.info( message, diff --git a/uvicorn/main.py b/uvicorn/main.py index da89ad184..9433a4740 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -10,7 +10,6 @@ import time import typing from email.utils import formatdate -from ipaddress import IPv4Address, IPv6Address, ip_address import click @@ -25,7 +24,6 @@ SSL_PROTOCOL_VERSION, WS_PROTOCOLS, Config, - _get_server_start_message, ) from uvicorn.supervisors import ChangeReload, Multiprocess @@ -506,6 +504,11 @@ def _share_socket(sock: socket) -> socket: else: # Standard case. Create a socket from a host/port pair. + addr_format = "%s://%s:%d" + if config.host and ":" in config.host: + # It's an IPv6 address. + addr_format = "%s://[%s]:%d" + try: server = await loop.create_server( create_protocol, @@ -522,17 +525,12 @@ def _share_socket(sock: socket) -> socket: if port == 0: port = server.sockets[0].getsockname()[1] protocol_name = "https" if config.ssl else "http" - try: - addr = ip_address(config.host) - if isinstance(addr, IPv6Address): - message, color_message = _get_server_start_message( - is_ipv6_message=True - ) - elif isinstance(addr, IPv4Address): - message, color_message = _get_server_start_message() - except ValueError: - message, color_message = _get_server_start_message() - + message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" + color_message = ( + "Uvicorn running on " + + click.style(addr_format, bold=True) + + " (Press CTRL+C to quit)" + ) logger.info( message, protocol_name, From 634aec95269b970940d39d1949baedd1dc347607 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sun, 8 Nov 2020 21:24:30 +0100 Subject: [PATCH 068/128] Isolate server code into an asyncio-specific module (#842) --- uvicorn/_impl/__init__.py | 0 uvicorn/_impl/asyncio.py | 256 ++++++++++++++++++++++++++++++++++++++ uvicorn/main.py | 254 +------------------------------------ 3 files changed, 261 insertions(+), 249 deletions(-) create mode 100644 uvicorn/_impl/__init__.py create mode 100644 uvicorn/_impl/asyncio.py diff --git a/uvicorn/_impl/__init__.py b/uvicorn/_impl/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/uvicorn/_impl/asyncio.py b/uvicorn/_impl/asyncio.py new file mode 100644 index 000000000..3331fcc4f --- /dev/null +++ b/uvicorn/_impl/asyncio.py @@ -0,0 +1,256 @@ +import asyncio +import functools +import logging +import os +import platform +import signal +import socket +import sys +import time +from email.utils import formatdate + +import click + +HANDLED_SIGNALS = ( + signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. + signal.SIGTERM, # Unix signal 15. Sent by `kill `. +) + +logger = logging.getLogger("uvicorn.error") + + +class AsyncioServerState: + """ + Shared servers state that is available between all protocol instances. + """ + + def __init__(self): + self.total_requests = 0 + self.connections = set() + self.tasks = set() + self.default_headers = [] + + +class AsyncioServer: + def __init__(self, config): + self.config = config + self.server_state = AsyncioServerState() + + self.started = False + self.should_exit = False + self.force_exit = False + self.last_notified = 0 + + def run(self, sockets=None): + self.config.setup_event_loop() + loop = asyncio.get_event_loop() + loop.run_until_complete(self.serve(sockets=sockets)) + + async def serve(self, sockets=None): + process_id = os.getpid() + + config = self.config + if not config.loaded: + config.load() + + self.lifespan = config.lifespan_class(config) + + self.install_signal_handlers() + + message = "Started server process [%d]" + color_message = "Started server process [" + click.style("%d", fg="cyan") + "]" + logger.info(message, process_id, extra={"color_message": color_message}) + + await self.startup(sockets=sockets) + if self.should_exit: + return + await self.main_loop() + await self.shutdown(sockets=sockets) + + message = "Finished server process [%d]" + color_message = "Finished server process [" + click.style("%d", fg="cyan") + "]" + logger.info( + "Finished server process [%d]", + process_id, + extra={"color_message": color_message}, + ) + + async def startup(self, sockets=None): + await self.lifespan.startup() + if self.lifespan.should_exit: + self.should_exit = True + return + + config = self.config + + create_protocol = functools.partial( + config.http_protocol_class, config=config, server_state=self.server_state + ) + + loop = asyncio.get_event_loop() + + if sockets is not None: + # Explicitly passed a list of open sockets. + # We use this when the server is run from a Gunicorn worker. + + def _share_socket(sock: socket) -> socket: + # Windows requires the socket be explicitly shared across + # multiple workers (processes). + from socket import fromshare # type: ignore + + sock_data = sock.share(os.getpid()) # type: ignore + return fromshare(sock_data) + + self.servers = [] + for sock in sockets: + if config.workers > 1 and platform.system() == "Windows": + sock = _share_socket(sock) + server = await loop.create_server( + create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog + ) + self.servers.append(server) + + elif config.fd is not None: + # Use an existing socket, from a file descriptor. + sock = socket.fromfd(config.fd, socket.AF_UNIX, socket.SOCK_STREAM) + server = await loop.create_server( + create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog + ) + message = "Uvicorn running on socket %s (Press CTRL+C to quit)" + logger.info(message % str(sock.getsockname())) + self.servers = [server] + + elif config.uds is not None: + # Create a socket using UNIX domain socket. + uds_perms = 0o666 + if os.path.exists(config.uds): + uds_perms = os.stat(config.uds).st_mode + server = await loop.create_unix_server( + create_protocol, path=config.uds, ssl=config.ssl, backlog=config.backlog + ) + os.chmod(config.uds, uds_perms) + message = "Uvicorn running on unix socket %s (Press CTRL+C to quit)" + logger.info(message % config.uds) + self.servers = [server] + + else: + # Standard case. Create a socket from a host/port pair. + addr_format = "%s://%s:%d" + if config.host and ":" in config.host: + # It's an IPv6 address. + addr_format = "%s://[%s]:%d" + + try: + server = await loop.create_server( + create_protocol, + host=config.host, + port=config.port, + ssl=config.ssl, + backlog=config.backlog, + ) + except OSError as exc: + logger.error(exc) + await self.lifespan.shutdown() + sys.exit(1) + port = config.port + if port == 0: + port = server.sockets[0].getsockname()[1] + protocol_name = "https" if config.ssl else "http" + message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" + color_message = ( + "Uvicorn running on " + + click.style(addr_format, bold=True) + + " (Press CTRL+C to quit)" + ) + logger.info( + message, + protocol_name, + config.host, + port, + extra={"color_message": color_message}, + ) + self.servers = [server] + + self.started = True + + async def main_loop(self): + counter = 0 + should_exit = await self.on_tick(counter) + while not should_exit: + counter += 1 + counter = counter % 864000 + await asyncio.sleep(0.1) + should_exit = await self.on_tick(counter) + + async def on_tick(self, counter) -> bool: + # Update the default headers, once per second. + if counter % 10 == 0: + current_time = time.time() + current_date = formatdate(current_time, usegmt=True).encode() + self.server_state.default_headers = [ + (b"date", current_date) + ] + self.config.encoded_headers + + # Callback to `callback_notify` once every `timeout_notify` seconds. + if self.config.callback_notify is not None: + if current_time - self.last_notified > self.config.timeout_notify: + self.last_notified = current_time + await self.config.callback_notify() + + # Determine if we should exit. + if self.should_exit: + return True + if self.config.limit_max_requests is not None: + return self.server_state.total_requests >= self.config.limit_max_requests + return False + + async def shutdown(self, sockets=None): + logger.info("Shutting down") + + # Stop accepting new connections. + for server in self.servers: + server.close() + for sock in sockets or []: + sock.close() + for server in self.servers: + await server.wait_closed() + + # Request shutdown on all existing connections. + for connection in list(self.server_state.connections): + connection.shutdown() + await asyncio.sleep(0.1) + + # Wait for existing connections to finish sending responses. + if self.server_state.connections and not self.force_exit: + msg = "Waiting for connections to close. (CTRL+C to force quit)" + logger.info(msg) + while self.server_state.connections and not self.force_exit: + await asyncio.sleep(0.1) + + # Wait for existing tasks to complete. + if self.server_state.tasks and not self.force_exit: + msg = "Waiting for background tasks to complete. (CTRL+C to force quit)" + logger.info(msg) + while self.server_state.tasks and not self.force_exit: + await asyncio.sleep(0.1) + + # Send the lifespan shutdown event, and wait for application shutdown. + if not self.force_exit: + await self.lifespan.shutdown() + + def install_signal_handlers(self): + loop = asyncio.get_event_loop() + + try: + for sig in HANDLED_SIGNALS: + loop.add_signal_handler(sig, self.handle_exit, sig, None) + except NotImplementedError: + # Windows + for sig in HANDLED_SIGNALS: + signal.signal(sig, self.handle_exit) + + def handle_exit(self, sig, frame): + if self.should_exit: + self.force_exit = True + else: + self.should_exit = True diff --git a/uvicorn/main.py b/uvicorn/main.py index 9433a4740..eaad7613f 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -1,19 +1,13 @@ -import asyncio -import functools import logging -import os import platform -import signal -import socket import ssl import sys -import time import typing -from email.utils import formatdate import click import uvicorn +from uvicorn._impl.asyncio import AsyncioServer, AsyncioServerState from uvicorn.config import ( HTTP_PROTOCOLS, INTERFACES, @@ -34,13 +28,12 @@ LOOP_CHOICES = click.Choice([key for key in LOOP_SETUPS.keys() if key != "none"]) INTERFACE_CHOICES = click.Choice(INTERFACES) -HANDLED_SIGNALS = ( - signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. - signal.SIGTERM, # Unix signal 15. Sent by `kill `. -) - logger = logging.getLogger("uvicorn.error") +# Aliases for backwards compatibility. These used to be defined here. +Server = AsyncioServer +ServerState = AsyncioServerState + def print_version(ctx, param, value): if not value or ctx.resilient_parsing: @@ -388,242 +381,5 @@ def run(app, **kwargs): server.run() -class ServerState: - """ - Shared servers state that is available between all protocol instances. - """ - - def __init__(self): - self.total_requests = 0 - self.connections = set() - self.tasks = set() - self.default_headers = [] - - -class Server: - def __init__(self, config): - self.config = config - self.server_state = ServerState() - - self.started = False - self.should_exit = False - self.force_exit = False - self.last_notified = 0 - - def run(self, sockets=None): - self.config.setup_event_loop() - loop = asyncio.get_event_loop() - loop.run_until_complete(self.serve(sockets=sockets)) - - async def serve(self, sockets=None): - process_id = os.getpid() - - config = self.config - if not config.loaded: - config.load() - - self.lifespan = config.lifespan_class(config) - - self.install_signal_handlers() - - message = "Started server process [%d]" - color_message = "Started server process [" + click.style("%d", fg="cyan") + "]" - logger.info(message, process_id, extra={"color_message": color_message}) - - await self.startup(sockets=sockets) - if self.should_exit: - return - await self.main_loop() - await self.shutdown(sockets=sockets) - - message = "Finished server process [%d]" - color_message = "Finished server process [" + click.style("%d", fg="cyan") + "]" - logger.info( - "Finished server process [%d]", - process_id, - extra={"color_message": color_message}, - ) - - async def startup(self, sockets=None): - await self.lifespan.startup() - if self.lifespan.should_exit: - self.should_exit = True - return - - config = self.config - - create_protocol = functools.partial( - config.http_protocol_class, config=config, server_state=self.server_state - ) - - loop = asyncio.get_event_loop() - - if sockets is not None: - # Explicitly passed a list of open sockets. - # We use this when the server is run from a Gunicorn worker. - - def _share_socket(sock: socket) -> socket: - # Windows requires the socket be explicitly shared across - # multiple workers (processes). - from socket import fromshare # type: ignore - - sock_data = sock.share(os.getpid()) # type: ignore - return fromshare(sock_data) - - self.servers = [] - for sock in sockets: - if config.workers > 1 and platform.system() == "Windows": - sock = _share_socket(sock) - server = await loop.create_server( - create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog - ) - self.servers.append(server) - - elif config.fd is not None: - # Use an existing socket, from a file descriptor. - sock = socket.fromfd(config.fd, socket.AF_UNIX, socket.SOCK_STREAM) - server = await loop.create_server( - create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog - ) - message = "Uvicorn running on socket %s (Press CTRL+C to quit)" - logger.info(message % str(sock.getsockname())) - self.servers = [server] - - elif config.uds is not None: - # Create a socket using UNIX domain socket. - uds_perms = 0o666 - if os.path.exists(config.uds): - uds_perms = os.stat(config.uds).st_mode - server = await loop.create_unix_server( - create_protocol, path=config.uds, ssl=config.ssl, backlog=config.backlog - ) - os.chmod(config.uds, uds_perms) - message = "Uvicorn running on unix socket %s (Press CTRL+C to quit)" - logger.info(message % config.uds) - self.servers = [server] - - else: - # Standard case. Create a socket from a host/port pair. - addr_format = "%s://%s:%d" - if config.host and ":" in config.host: - # It's an IPv6 address. - addr_format = "%s://[%s]:%d" - - try: - server = await loop.create_server( - create_protocol, - host=config.host, - port=config.port, - ssl=config.ssl, - backlog=config.backlog, - ) - except OSError as exc: - logger.error(exc) - await self.lifespan.shutdown() - sys.exit(1) - port = config.port - if port == 0: - port = server.sockets[0].getsockname()[1] - protocol_name = "https" if config.ssl else "http" - message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" - color_message = ( - "Uvicorn running on " - + click.style(addr_format, bold=True) - + " (Press CTRL+C to quit)" - ) - logger.info( - message, - protocol_name, - config.host, - port, - extra={"color_message": color_message}, - ) - self.servers = [server] - - self.started = True - - async def main_loop(self): - counter = 0 - should_exit = await self.on_tick(counter) - while not should_exit: - counter += 1 - counter = counter % 864000 - await asyncio.sleep(0.1) - should_exit = await self.on_tick(counter) - - async def on_tick(self, counter) -> bool: - # Update the default headers, once per second. - if counter % 10 == 0: - current_time = time.time() - current_date = formatdate(current_time, usegmt=True).encode() - self.server_state.default_headers = [ - (b"date", current_date) - ] + self.config.encoded_headers - - # Callback to `callback_notify` once every `timeout_notify` seconds. - if self.config.callback_notify is not None: - if current_time - self.last_notified > self.config.timeout_notify: - self.last_notified = current_time - await self.config.callback_notify() - - # Determine if we should exit. - if self.should_exit: - return True - if self.config.limit_max_requests is not None: - return self.server_state.total_requests >= self.config.limit_max_requests - return False - - async def shutdown(self, sockets=None): - logger.info("Shutting down") - - # Stop accepting new connections. - for server in self.servers: - server.close() - for sock in sockets or []: - sock.close() - for server in self.servers: - await server.wait_closed() - - # Request shutdown on all existing connections. - for connection in list(self.server_state.connections): - connection.shutdown() - await asyncio.sleep(0.1) - - # Wait for existing connections to finish sending responses. - if self.server_state.connections and not self.force_exit: - msg = "Waiting for connections to close. (CTRL+C to force quit)" - logger.info(msg) - while self.server_state.connections and not self.force_exit: - await asyncio.sleep(0.1) - - # Wait for existing tasks to complete. - if self.server_state.tasks and not self.force_exit: - msg = "Waiting for background tasks to complete. (CTRL+C to force quit)" - logger.info(msg) - while self.server_state.tasks and not self.force_exit: - await asyncio.sleep(0.1) - - # Send the lifespan shutdown event, and wait for application shutdown. - if not self.force_exit: - await self.lifespan.shutdown() - - def install_signal_handlers(self): - loop = asyncio.get_event_loop() - - try: - for sig in HANDLED_SIGNALS: - loop.add_signal_handler(sig, self.handle_exit, sig, None) - except NotImplementedError: - # Windows - for sig in HANDLED_SIGNALS: - signal.signal(sig, self.handle_exit) - - def handle_exit(self, sig, frame): - if self.should_exit: - self.force_exit = True - else: - self.should_exit = True - - if __name__ == "__main__": main() From 45322cc4cee43c6857c40c424a186ef903a70f96 Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 9 Nov 2020 19:29:50 +0100 Subject: [PATCH 069/128] Fix protocol not imported directly (#838) --- tests/test_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 7428b31d9..9499a849f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,11 +5,11 @@ import pytest import yaml -from uvicorn import protocols from uvicorn.config import LOGGING_CONFIG, Config from uvicorn.middleware.debug import DebugMiddleware from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware from uvicorn.middleware.wsgi import WSGIMiddleware +from uvicorn.protocols.http.h11_impl import H11Protocol @pytest.fixture @@ -72,9 +72,9 @@ def test_app_unimportable(): def test_concrete_http_class(): - config = Config(app=asgi_app, http=protocols.http.h11_impl.H11Protocol) + config = Config(app=asgi_app, http=H11Protocol) config.load() - assert config.http_protocol_class is protocols.http.h11_impl.H11Protocol + assert config.http_protocol_class is H11Protocol def test_socket_bind(): From 45e6e83100fda7d59790eb1b3518a42fe9bafea9 Mon Sep 17 00:00:00 2001 From: Tristan King Date: Thu, 12 Nov 2020 09:59:03 +0100 Subject: [PATCH 070/128] Use latin1 when decoding X-Forwarded-* headers (#701) --- tests/middleware/test_proxy_headers.py | 11 +++++++++++ uvicorn/middleware/proxy_headers.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/middleware/test_proxy_headers.py b/tests/middleware/test_proxy_headers.py index 8ed1e9d1f..eef8dec5a 100644 --- a/tests/middleware/test_proxy_headers.py +++ b/tests/middleware/test_proxy_headers.py @@ -28,3 +28,14 @@ def test_proxy_headers_no_port(): response = client.get("/", headers=headers) assert response.status_code == 200 assert response.text == "Remote: https://1.2.3.4:0" + + +def test_proxy_headers_invalid_x_forwarded_for(): + client = TestClient(app) + headers = { + "X-Forwarded-Proto": "https", + "X-Forwarded-For": "\xf0\xfd\xfd\xfd, 1.2.3.4", + } + response = client.get("/", headers=headers) + assert response.status_code == 200 + assert response.text == "Remote: https://1.2.3.4:0" diff --git a/uvicorn/middleware/proxy_headers.py b/uvicorn/middleware/proxy_headers.py index e0d75fd08..05f4f70e1 100644 --- a/uvicorn/middleware/proxy_headers.py +++ b/uvicorn/middleware/proxy_headers.py @@ -30,14 +30,14 @@ async def __call__(self, scope, receive, send): if b"x-forwarded-proto" in headers: # Determine if the incoming request was http or https based on # the X-Forwarded-Proto header. - x_forwarded_proto = headers[b"x-forwarded-proto"].decode("ascii") + x_forwarded_proto = headers[b"x-forwarded-proto"].decode("latin1") scope["scheme"] = x_forwarded_proto.strip() if b"x-forwarded-for" in headers: # Determine the client address from the last trusted IP in the # X-Forwarded-For header. We've lost the connecting client's port # information by now, so only include the host. - x_forwarded_for = headers[b"x-forwarded-for"].decode("ascii") + x_forwarded_for = headers[b"x-forwarded-for"].decode("latin1") host = x_forwarded_for.split(",")[-1].strip() port = 0 scope["client"] = (host, port) From de213614b7f8309f411e30a31d274d01b129607d Mon Sep 17 00:00:00 2001 From: euri10 Date: Wed, 18 Nov 2020 20:56:09 +0100 Subject: [PATCH 071/128] Fix race condition that leads Quart to hang with uvicorn (#848) * Revert "Cancel old keepalive-trigger before setting new one. (#832)" This reverts commit d5dcf80c * Revert "Revert "Cancel old keepalive-trigger before setting new one. (#832)"" This reverts commit 64049e55 * App that reproduce issue with quart * Trim logs * Added diff logs * Added lua payload * Less diff * Add pure asgi app, cant reproduce * Set event per cycle * Removed log trace * Message event is now in cycle * Removed logs * Cannot set message if cycle is None * Deleted logs * Test concurrent hang * Hanging test * Right test and modified client a little to be able to use context manager * emoved test client changes and test * Add event set in send coroutine * Tests leftovers * h11 implementation * Removed working files * Suggestions implemented --- uvicorn/protocols/http/h11_impl.py | 11 ++++++----- uvicorn/protocols/http/httptools_impl.py | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index 6a362ff20..e9583d70b 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -113,7 +113,6 @@ def __init__(self, config, server_state, _loop=None): self.scope = None self.headers = None self.cycle = None - self.message_event = asyncio.Event() # Protocol interface def connection_made(self, transport): @@ -146,7 +145,8 @@ def connection_lost(self, exc): # Premature client disconnect pass - self.message_event.set() + if self.cycle is not None: + self.cycle.message_event.set() if self.flow is not None: self.flow.resume_writing() @@ -234,7 +234,7 @@ def handle_events(self): access_logger=self.access_logger, access_log=self.access_log, default_headers=self.default_headers, - message_event=self.message_event, + message_event=asyncio.Event(), on_response=self.on_response_complete, ) task = self.loop.create_task(self.cycle.run_asgi(app)) @@ -247,7 +247,7 @@ def handle_events(self): self.cycle.body += event.data if len(self.cycle.body) > HIGH_WATER_LIMIT: self.flow.pause_reading() - self.message_event.set() + self.cycle.message_event.set() elif event_type is h11.EndOfMessage: if self.conn.our_state is h11.DONE: @@ -255,7 +255,7 @@ def handle_events(self): self.conn.start_next_cycle() continue self.cycle.more_body = False - self.message_event.set() + self.cycle.message_event.set() def handle_upgrade(self, event): upgrade_value = None @@ -491,6 +491,7 @@ async def send(self, message): # Handle response completion if not more_body: self.response_complete = True + self.message_event.set() event = h11.EndOfMessage() output = self.conn.send(event) self.transport.write(output) diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index 15262326b..924e131ce 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -121,7 +121,6 @@ def __init__(self, config, server_state, _loop=None): self.headers = None self.expect_100_continue = False self.cycle = None - self.message_event = asyncio.Event() # Protocol interface def connection_made(self, transport): @@ -146,7 +145,8 @@ def connection_lost(self, exc): if self.cycle and not self.cycle.response_complete: self.cycle.disconnected = True - self.message_event.set() + if self.cycle is not None: + self.cycle.message_event.set() if self.flow is not None: self.flow.resume_writing() @@ -267,7 +267,7 @@ def on_headers_complete(self): access_logger=self.access_logger, access_log=self.access_log, default_headers=self.default_headers, - message_event=self.message_event, + message_event=asyncio.Event(), expect_100_continue=self.expect_100_continue, keep_alive=http_version != "1.0", on_response=self.on_response_complete, @@ -288,13 +288,13 @@ def on_body(self, body: bytes): self.cycle.body += body if len(self.cycle.body) > HIGH_WATER_LIMIT: self.flow.pause_reading() - self.message_event.set() + self.cycle.message_event.set() def on_message_complete(self): if self.parser.should_upgrade() or self.cycle.response_complete: return self.cycle.more_body = False - self.message_event.set() + self.cycle.message_event.set() def on_response_complete(self): # Callback for pipelined HTTP requests to be started. @@ -530,6 +530,7 @@ async def send(self, message): if self.expected_content_length != 0: raise RuntimeError("Response content shorter than Content-Length") self.response_complete = True + self.message_event.set() if not self.keep_alive: self.transport.close() self.on_response() From 79a72e3dc97db4803ea4e5c92c936fd45f885b38 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Wed, 18 Nov 2020 22:07:38 +0100 Subject: [PATCH 072/128] Modernize test ASGI applications (#854) --- tests/middleware/test_trace_logging.py | 28 ++++---------- tests/protocols/test_websocket.py | 32 +++++----------- tests/test_default_headers.py | 20 ++++------ tests/test_main.py | 53 +++++++------------------- tests/test_ssl.py | 39 +++++-------------- 5 files changed, 49 insertions(+), 123 deletions(-) diff --git a/tests/middleware/test_trace_logging.py b/tests/middleware/test_trace_logging.py index 71ef82398..762b71fd1 100644 --- a/tests/middleware/test_trace_logging.py +++ b/tests/middleware/test_trace_logging.py @@ -48,26 +48,23 @@ } +async def app(scope, receive, send): + assert scope["type"] == "http" + await send({"type": "http.response.start", "status": 204, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + @pytest.mark.skipif( sys.platform.startswith("win") or platform.python_implementation() == "PyPy", reason="Skipping test on Windows and PyPy", ) def test_trace_logging(capsys): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, log_config=test_logging_config, @@ -92,21 +89,12 @@ def install_signal_handlers(self): ) @pytest.mark.parametrize("http_protocol", [("h11"), ("httptools")]) def test_access_logging(capsys, http_protocol): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass config = Config( - app=App, + app=app, loop="asyncio", http=http_protocol, limit_max_requests=1, diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 98445de2a..71b81fcd2 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -316,17 +316,13 @@ async def get_data(url): @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) def test_missing_handshake(protocol_cls): - class App: - def __init__(self, scope): - pass - - async def __call__(self, receive, send): - pass + async def app(app, receive, send): + pass async def connect(url): await websockets.connect(url) - with run_server(App, protocol_cls=protocol_cls) as url: + with run_server(app, protocol_cls=protocol_cls) as url: loop = asyncio.new_event_loop() with pytest.raises(websockets.exceptions.InvalidStatusCode) as exc_info: loop.run_until_complete(connect(url)) @@ -336,17 +332,13 @@ async def connect(url): @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) def test_send_before_handshake(protocol_cls): - class App: - def __init__(self, scope): - pass - - async def __call__(self, receive, send): - await send({"type": "websocket.send", "text": "123"}) + async def app(scope, receive, send): + await send({"type": "websocket.send", "text": "123"}) async def connect(url): await websockets.connect(url) - with run_server(App, protocol_cls=protocol_cls) as url: + with run_server(app, protocol_cls=protocol_cls) as url: loop = asyncio.new_event_loop() with pytest.raises(websockets.exceptions.InvalidStatusCode) as exc_info: loop.run_until_complete(connect(url)) @@ -356,19 +348,15 @@ async def connect(url): @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) def test_duplicate_handshake(protocol_cls): - class App: - def __init__(self, scope): - pass - - async def __call__(self, receive, send): - await send({"type": "websocket.accept"}) - await send({"type": "websocket.accept"}) + async def app(scope, receive, send): + await send({"type": "websocket.accept"}) + await send({"type": "websocket.accept"}) async def connect(url): async with websockets.connect(url) as websocket: _ = await websocket.recv() - with run_server(App, protocol_cls=protocol_cls) as url: + with run_server(app, protocol_cls=protocol_cls) as url: loop = asyncio.new_event_loop() with pytest.raises(websockets.exceptions.ConnectionClosed) as exc_info: loop.run_until_complete(connect(url)) diff --git a/tests/test_default_headers.py b/tests/test_default_headers.py index b3c029efe..b9ed98188 100644 --- a/tests/test_default_headers.py +++ b/tests/test_default_headers.py @@ -6,14 +6,10 @@ from uvicorn import Config, Server -class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 200, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) +async def app(scope, receive, send): + assert scope["type"] == "http" + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) class CustomServer(Server): @@ -22,7 +18,7 @@ def install_signal_handlers(self): def test_default_default_headers(): - config = Config(app=App, loop="asyncio", limit_max_requests=1) + config = Config(app=app, loop="asyncio", limit_max_requests=1) server = CustomServer(config=config) thread = threading.Thread(target=server.run) thread.start() @@ -37,7 +33,7 @@ def test_default_default_headers(): def test_override_server_header(): config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, headers=[("Server", "over-ridden")], @@ -56,7 +52,7 @@ def test_override_server_header(): def test_override_server_header_multiple_times(): config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, headers=[("Server", "over-ridden"), ("Server", "another-value")], @@ -78,7 +74,7 @@ def test_override_server_header_multiple_times(): def test_add_additional_header(): config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, headers=[("X-Additional", "new-value")], diff --git a/tests/test_main.py b/tests/test_main.py index a04178f14..629a8de84 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -9,6 +9,12 @@ from uvicorn.main import Server +async def app(scope, receive, send): + assert scope["type"] == "http" + await send({"type": "http.response.start", "status": 204, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + @pytest.mark.parametrize( "host, url", [ @@ -18,20 +24,11 @@ ], ) def test_run(host, url): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass - config = Config(app=App, host=host, loop="asyncio", limit_max_requests=1) + config = Config(app=app, host=host, loop="asyncio", limit_max_requests=1) server = CustomServer(config=config) thread = threading.Thread(target=server.run) thread.start() @@ -43,20 +40,11 @@ def install_signal_handlers(self): def test_run_multiprocess(): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass - config = Config(app=App, loop="asyncio", workers=2, limit_max_requests=1) + config = Config(app=app, loop="asyncio", workers=2, limit_max_requests=1) server = CustomServer(config=config) thread = threading.Thread(target=server.run) thread.start() @@ -68,20 +56,11 @@ def install_signal_handlers(self): def test_run_reload(): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass - config = Config(app=App, loop="asyncio", reload=True, limit_max_requests=1) + config = Config(app=app, loop="asyncio", reload=True, limit_max_requests=1) server = CustomServer(config=config) thread = threading.Thread(target=server.run) thread.start() @@ -93,20 +72,16 @@ def install_signal_handlers(self): def test_run_with_shutdown(): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - while True: - time.sleep(1) + async def app(scope, receive, send): + assert scope["type"] == "http" + while True: + time.sleep(1) class CustomServer(Server): def install_signal_handlers(self): pass - config = Config(app=App, loop="asyncio", workers=2, limit_max_requests=1) + config = Config(app=app, loop="asyncio", workers=2, limit_max_requests=1) server = CustomServer(config=config) sock = config.bind_socket() exc = True diff --git a/tests/test_ssl.py b/tests/test_ssl.py index d64d828ab..5bd5dc73d 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -25,25 +25,22 @@ def no_ssl_verification(session=requests.Session): session.request = old_request +async def app(scope, receive, send): + assert scope["type"] == "http" + await send({"type": "http.response.start", "status": 204, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + @pytest.mark.skipif( sys.platform.startswith("win"), reason="Skipping SSL test on Windows" ) def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, ssl_keyfile=tls_ca_certificate_private_key_path, @@ -64,21 +61,12 @@ def install_signal_handlers(self): sys.platform.startswith("win"), reason="Skipping SSL test on Windows" ) def test_run_chain(tls_certificate_pem_path): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, ssl_certfile=tls_certificate_pem_path, @@ -100,21 +88,12 @@ def install_signal_handlers(self): def test_run_password( tls_ca_certificate_pem_path, tls_ca_certificate_private_key_encrypted_path ): - class App: - def __init__(self, scope): - if scope["type"] != "http": - raise Exception() - - async def __call__(self, receive, send): - await send({"type": "http.response.start", "status": 204, "headers": []}) - await send({"type": "http.response.body", "body": b"", "more_body": False}) - class CustomServer(Server): def install_signal_handlers(self): pass config = Config( - app=App, + app=app, loop="asyncio", limit_max_requests=1, ssl_keyfile=tls_ca_certificate_private_key_encrypted_path, From 0b941abfbb527d102d11e950c2fea1f9ecf1988f Mon Sep 17 00:00:00 2001 From: euri10 Date: Sun, 22 Nov 2020 10:31:45 +0100 Subject: [PATCH 073/128] v0.12.3 (#862) * Completed CHANGELOG.md and new bimp * Modified after reading https://keepachangelog.com/ * bullet points! Co-authored-by: Florimond Manca Co-authored-by: Florimond Manca --- CHANGELOG.md | 8 ++++++++ uvicorn/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8367d22bc..f47c8ba78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log +## 0.12.3 - 2020-11-21 + +### Fixed +- Fix race condition that leads Quart to hang with uvicorn (#848) 11/18/20 de213614 +- Use latin1 when decoding X-Forwarded-* headers (#701) 11/12/20 45e6e831 +- Rework IPv6 support (#837) 11/8/20 bdab488e +- Cancel old keepalive-trigger before setting new one. (#832) 10/26/20 d5dcf80c + ## 0.12.2 - 2020-10-19 ### Added diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 2e4d85ce1..d2929d860 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.12.2" +__version__ = "0.12.3" __all__ = ["main", "run", "Config", "Server"] From 16d12f3aea1df4e1b82bdb7054d82e2a0df42da0 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Tue, 24 Nov 2020 09:44:07 +0100 Subject: [PATCH 074/128] Fix typo in settings documentation (#864) --- docs/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings.md b/docs/settings.md index dc5682e24..684098a90 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -58,7 +58,7 @@ Note that WSGI mode always disables WebSocket support, as it is not supported by * `--root-path ` - Set the ASGI `root_path` for applications submounted below a given URL path. * `--proxy-headers` / `--no-proxy-headers` - Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info. Defaults to enabled, but is restricted to only trusting connecting IPs in the `forwarded-allow-ips` configuration. -* `--forwarded-allow-ips` Comma separated list of IPs to trust with proxy headers. Defaults to the ``$FORWARDED_ALLOW_IPS` environment variable if available, or '127.0.0.1'. A wildcard '*' means always trust. +* `--forwarded-allow-ips` Comma separated list of IPs to trust with proxy headers. Defaults to the `$FORWARDED_ALLOW_IPS` environment variable if available, or '127.0.0.1'. A wildcard '*' means always trust. ## HTTPS From f1bcade4d55eb7f7120aba9bf24ccb863c1502d8 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Wed, 25 Nov 2020 13:23:21 +0100 Subject: [PATCH 075/128] Move server implementation to server.py (#866) * Move server implementation to server.py * Lint --- uvicorn/_impl/__init__.py | 0 uvicorn/main.py | 6 +----- uvicorn/{_impl/asyncio.py => server.py} | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 uvicorn/_impl/__init__.py rename uvicorn/{_impl/asyncio.py => server.py} (98%) diff --git a/uvicorn/_impl/__init__.py b/uvicorn/_impl/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/uvicorn/main.py b/uvicorn/main.py index eaad7613f..ccbe9c17e 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -7,7 +7,6 @@ import click import uvicorn -from uvicorn._impl.asyncio import AsyncioServer, AsyncioServerState from uvicorn.config import ( HTTP_PROTOCOLS, INTERFACES, @@ -19,6 +18,7 @@ WS_PROTOCOLS, Config, ) +from uvicorn.server import Server, ServerState # noqa: F401 # Used to be defined here. from uvicorn.supervisors import ChangeReload, Multiprocess LEVEL_CHOICES = click.Choice(LOG_LEVELS.keys()) @@ -30,10 +30,6 @@ logger = logging.getLogger("uvicorn.error") -# Aliases for backwards compatibility. These used to be defined here. -Server = AsyncioServer -ServerState = AsyncioServerState - def print_version(ctx, param, value): if not value or ctx.resilient_parsing: diff --git a/uvicorn/_impl/asyncio.py b/uvicorn/server.py similarity index 98% rename from uvicorn/_impl/asyncio.py rename to uvicorn/server.py index 3331fcc4f..1e5981be9 100644 --- a/uvicorn/_impl/asyncio.py +++ b/uvicorn/server.py @@ -19,7 +19,7 @@ logger = logging.getLogger("uvicorn.error") -class AsyncioServerState: +class ServerState: """ Shared servers state that is available between all protocol instances. """ @@ -31,10 +31,10 @@ def __init__(self): self.default_headers = [] -class AsyncioServer: +class Server: def __init__(self, config): self.config = config - self.server_state = AsyncioServerState() + self.server_state = ServerState() self.started = False self.should_exit = False From ce2ef45a9109df8eae038c0ec323eb63d644cbc6 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 7 Dec 2020 21:50:59 +0100 Subject: [PATCH 076/128] Skip installation of signal handlers when not in main thread (#871) * Skip installation of signal handlers when not in main thread * Update uvicorn/server.py Co-authored-by: euri10 Co-authored-by: euri10 --- tests/middleware/test_trace_logging.py | 12 ++---------- tests/test_default_headers.py | 13 ++++--------- tests/test_main.py | 24 ++++-------------------- tests/test_ssl.py | 18 +++--------------- uvicorn/server.py | 7 ++++++- 5 files changed, 19 insertions(+), 55 deletions(-) diff --git a/tests/middleware/test_trace_logging.py b/tests/middleware/test_trace_logging.py index 762b71fd1..ce3c37f27 100644 --- a/tests/middleware/test_trace_logging.py +++ b/tests/middleware/test_trace_logging.py @@ -59,10 +59,6 @@ async def app(scope, receive, send): reason="Skipping test on Windows and PyPy", ) def test_trace_logging(capsys): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config( app=app, loop="asyncio", @@ -70,7 +66,7 @@ def install_signal_handlers(self): log_config=test_logging_config, log_level="trace", ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -89,10 +85,6 @@ def install_signal_handlers(self): ) @pytest.mark.parametrize("http_protocol", [("h11"), ("httptools")]) def test_access_logging(capsys, http_protocol): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config( app=app, loop="asyncio", @@ -100,7 +92,7 @@ def install_signal_handlers(self): limit_max_requests=1, log_config=test_logging_config, ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: diff --git a/tests/test_default_headers.py b/tests/test_default_headers.py index b9ed98188..435d64b61 100644 --- a/tests/test_default_headers.py +++ b/tests/test_default_headers.py @@ -12,14 +12,9 @@ async def app(scope, receive, send): await send({"type": "http.response.body", "body": b"", "more_body": False}) -class CustomServer(Server): - def install_signal_handlers(self): - pass - - def test_default_default_headers(): config = Config(app=app, loop="asyncio", limit_max_requests=1) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -38,7 +33,7 @@ def test_override_server_header(): limit_max_requests=1, headers=[("Server", "over-ridden")], ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -57,7 +52,7 @@ def test_override_server_header_multiple_times(): limit_max_requests=1, headers=[("Server", "over-ridden"), ("Server", "another-value")], ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -79,7 +74,7 @@ def test_add_additional_header(): limit_max_requests=1, headers=[("X-Additional", "new-value")], ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: diff --git a/tests/test_main.py b/tests/test_main.py index 629a8de84..845a9c9a6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -24,12 +24,8 @@ async def app(scope, receive, send): ], ) def test_run(host, url): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config(app=app, host=host, loop="asyncio", limit_max_requests=1) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -40,12 +36,8 @@ def install_signal_handlers(self): def test_run_multiprocess(): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config(app=app, loop="asyncio", workers=2, limit_max_requests=1) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -56,12 +48,8 @@ def install_signal_handlers(self): def test_run_reload(): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config(app=app, loop="asyncio", reload=True, limit_max_requests=1) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -77,12 +65,8 @@ async def app(scope, receive, send): while True: time.sleep(1) - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config(app=app, loop="asyncio", workers=2, limit_max_requests=1) - server = CustomServer(config=config) + server = Server(config=config) sock = config.bind_socket() exc = True diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 5bd5dc73d..19948ae4c 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -35,10 +35,6 @@ async def app(scope, receive, send): sys.platform.startswith("win"), reason="Skipping SSL test on Windows" ) def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config( app=app, loop="asyncio", @@ -46,7 +42,7 @@ def install_signal_handlers(self): ssl_keyfile=tls_ca_certificate_private_key_path, ssl_certfile=tls_ca_certificate_pem_path, ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -61,17 +57,13 @@ def install_signal_handlers(self): sys.platform.startswith("win"), reason="Skipping SSL test on Windows" ) def test_run_chain(tls_certificate_pem_path): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config( app=app, loop="asyncio", limit_max_requests=1, ssl_certfile=tls_certificate_pem_path, ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: @@ -88,10 +80,6 @@ def install_signal_handlers(self): def test_run_password( tls_ca_certificate_pem_path, tls_ca_certificate_private_key_encrypted_path ): - class CustomServer(Server): - def install_signal_handlers(self): - pass - config = Config( app=app, loop="asyncio", @@ -100,7 +88,7 @@ def install_signal_handlers(self): ssl_certfile=tls_ca_certificate_pem_path, ssl_keyfile_password="uvicorn password for the win", ) - server = CustomServer(config=config) + server = Server(config=config) thread = threading.Thread(target=server.run) thread.start() while not server.started: diff --git a/uvicorn/server.py b/uvicorn/server.py index 1e5981be9..d289ee599 100644 --- a/uvicorn/server.py +++ b/uvicorn/server.py @@ -6,6 +6,7 @@ import signal import socket import sys +import threading import time from email.utils import formatdate @@ -238,7 +239,11 @@ async def shutdown(self, sockets=None): if not self.force_exit: await self.lifespan.shutdown() - def install_signal_handlers(self): + def install_signal_handlers(self) -> None: + if threading.current_thread() is not threading.main_thread(): + # Signals can only be listened to from the main thread. + return + loop = asyncio.get_event_loop() try: From 50fc0d1c07cb8542fe6f73bd9473a6772f5d980f Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Mon, 7 Dec 2020 23:29:48 +0100 Subject: [PATCH 077/128] Add --factory flag to support factory-style application imports (#875) * Add --factory flag to support factory-style application imports * Tweak docs copy * Tighten loose test * Address feedback --- docs/index.md | 18 ++++++++++++++++++ docs/settings.md | 1 + tests/test_auto_detection.py | 8 ++++++-- tests/test_config.py | 23 +++++++++++++++++++++-- uvicorn/config.py | 15 +++++++++++++++ uvicorn/main.py | 9 +++++++++ 6 files changed, 70 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index ae89e0454..a91d59e59 100644 --- a/docs/index.md +++ b/docs/index.md @@ -158,6 +158,8 @@ Options: [default: TLSv1] --header TEXT Specify custom default HTTP response headers as a Name:Value pair + --factory Treat APP as an application factory, i.e. a + () -> function. [default: False] --help Show this message and exit. ``` @@ -200,6 +202,22 @@ For a [PyPy][pypy] compatible configuration use `uvicorn.workers.UvicornH11Worke For more information, see the [deployment documentation](deployment.md). +### Application factories + +The `--factory` flag allows loading the application from a factory function, rather than an application instance directly. The factory will be called with no arguments and should return an ASGI application. + +**example.py**: + +```python +def create_app(): + app = ... + return app +``` + +```shell +$ uvicorn --factory example:create_app +``` + ## The ASGI interface Uvicorn uses the [ASGI specification][asgi] for interacting with an application. diff --git a/docs/settings.md b/docs/settings.md index 684098a90..a3dad5c8c 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -8,6 +8,7 @@ equivalent keyword arguments, eg. `uvicorn.run("example:app", port=5000, reload= ## Application * `APP` - The ASGI application to run, in the format `":"`. +* `--factory` - Treat `APP` as an application factory, i.e. a `() -> ` callable. ## Socket Binding diff --git a/tests/test_auto_detection.py b/tests/test_auto_detection.py index dc34b11c8..fe05497c4 100644 --- a/tests/test_auto_detection.py +++ b/tests/test_auto_detection.py @@ -23,6 +23,10 @@ websockets = None +async def app(scope, receive, send): + pass # pragma: no cover + + # TODO: Add pypy to our testing matrix, and assert we get the correct classes # dependent on the platform we're running the tests under. @@ -36,7 +40,7 @@ def test_loop_auto(): def test_http_auto(): - config = Config(app=None) + config = Config(app=app) server_state = ServerState() protocol = AutoHTTPProtocol(config=config, server_state=server_state) expected_http = "H11Protocol" if httptools is None else "HttpToolsProtocol" @@ -44,7 +48,7 @@ def test_http_auto(): def test_websocket_auto(): - config = Config(app=None) + config = Config(app=app) server_state = ServerState() protocol = AutoWebSocketsProtocol(config=config, server_state=server_state) expected_websockets = "WSProtocol" if websockets is None else "WebSocketProtocol" diff --git a/tests/test_config.py b/tests/test_config.py index 9499a849f..67126f087 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -32,11 +32,11 @@ def yaml_logging_config(logging_config): return yaml.dump(logging_config) -async def asgi_app(): +async def asgi_app(scope, receive, send): pass # pragma: nocover -def wsgi_app(): +def wsgi_app(environ, start_response): pass # pragma: nocover @@ -71,6 +71,25 @@ def test_app_unimportable(): config.load() +def test_app_factory(): + def create_app(): + return asgi_app + + config = Config(app=create_app, factory=True, proxy_headers=False) + config.load() + assert config.loaded_app is asgi_app + + # Flag missing. + config = Config(app=create_app) + with pytest.raises(SystemExit): + config.load() + + # App not a no-arguments callable. + config = Config(app=asgi_app, factory=True) + with pytest.raises(SystemExit): + config.load() + + def test_concrete_http_class(): config = Config(app=asgi_app, http=H11Protocol) config.load() diff --git a/uvicorn/config.py b/uvicorn/config.py index 0b7bd1ef0..62fe3c5d0 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -155,6 +155,7 @@ def __init__( ssl_ca_certs=None, ssl_ciphers="TLSv1", headers=None, + factory=False, ): self.app = app self.host = host @@ -191,6 +192,7 @@ def __init__( self.ssl_ciphers = ssl_ciphers self.headers = headers if headers else [] # type: List[str] self.encoded_headers = None # type: List[Tuple[bytes, bytes]] + self.factory = factory self.loaded = False self.configure_logging() @@ -308,6 +310,19 @@ def load(self): logger.error("Error loading ASGI app. %s" % exc) sys.exit(1) + if self.factory: + try: + self.loaded_app = self.loaded_app() + except TypeError as exc: + logger.error("Error loading ASGI app factory: %s", exc) + sys.exit(1) + elif not inspect.signature(self.loaded_app).parameters: + logger.error( + "APP seems to be an application factory. " + "Run uvicorn with the --factory flag." + ) + sys.exit(1) + if self.interface == "auto": if inspect.isclass(self.loaded_app): use_asgi_3 = hasattr(self.loaded_app, "__await__") diff --git a/uvicorn/main.py b/uvicorn/main.py index ccbe9c17e..f82b98c0f 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -273,6 +273,13 @@ def print_version(ctx, param, value): help="Look for APP in the specified directory, by adding this to the PYTHONPATH." " Defaults to the current working directory.", ) +@click.option( + "--factory", + is_flag=True, + default=False, + help="Treat APP as an application factory, i.e. a () -> callable.", + show_default=True, +) def main( app, host: str, @@ -310,6 +317,7 @@ def main( headers: typing.List[str], use_colors: bool, app_dir: str, + factory: bool, ): sys.path.insert(0, app_dir) @@ -349,6 +357,7 @@ def main( "ssl_ciphers": ssl_ciphers, "headers": list([header.split(":", 1) for header in headers]), "use_colors": use_colors, + "factory": factory, } run(**kwargs) From 41a8be451da598d54b99c7ecf948bbd3937d8bda Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Tue, 8 Dec 2020 17:17:56 +0100 Subject: [PATCH 078/128] Version 0.13.0 (#874) --- CHANGELOG.md | 7 +++++++ uvicorn/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f47c8ba78..7820a3fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Change Log +## 0.13.0 - 2020-12-07 + +### Added + +- Add `--factory` flag to support factory-style application imports. (#875) 2020-12-07 50fc0d1c +- Skip installation of signal handlers when not in the main thread. Allows using `Server` in multithreaded contexts without having to override `.install_signal_handlers()`. (#871) 2020-12-07 ce2ef45a + ## 0.12.3 - 2020-11-21 ### Fixed diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index d2929d860..184b5a329 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.12.3" +__version__ = "0.13.0" __all__ = ["main", "run", "Config", "Server"] From 1b36d69d2145306a1696c6ff8fdf4f966eecab43 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Wed, 9 Dec 2020 09:35:29 +0100 Subject: [PATCH 079/128] Fix release date of 0.13.0 (#878) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7820a3fb8..5ed58460e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log -## 0.13.0 - 2020-12-07 +## 0.13.0 - 2020-12-08 ### Added From 5311e11fbedb8f39fd913b3ce305962aa476b76b Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Wed, 9 Dec 2020 09:36:09 +0100 Subject: [PATCH 080/128] Add tool to keep CLI usage in `index.md` in sync (#876) * Add tool to keep CLI usage in `index.md` in sync * Ensure final newline * Keep deployment.md in sync as well * Rename and reorganize tool --- docs/deployment.md | 39 ++++++++++++++++++++++---- docs/index.md | 43 +++++++++++++++++++++++----- scripts/check | 3 +- scripts/lint | 1 + tools/cli_usage.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++ uvicorn/main.py | 2 +- 6 files changed, 143 insertions(+), 15 deletions(-) create mode 100644 tools/cli_usage.py diff --git a/docs/deployment.md b/docs/deployment.md index 053c4f7e6..4ace0b73d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -23,6 +23,7 @@ When running locally, use `--reload` to turn on auto-reloading. To see the complete set of available options, use `uvicorn --help`: + ``` $ uvicorn --help Usage: uvicorn [OPTIONS] APP @@ -30,29 +31,39 @@ Usage: uvicorn [OPTIONS] APP Options: --host TEXT Bind socket to this host. [default: 127.0.0.1] + --port INTEGER Bind socket to this port. [default: 8000] --uds TEXT Bind to a UNIX domain socket. --fd INTEGER Bind to socket from this file descriptor. --reload Enable auto-reload. --reload-dir TEXT Set reload directories explicitly, instead of using the current working directory. + + --reload-delay FLOAT Delay between previous and next check if + application needs to be. Defaults to 0.25s. + [default: 0.25] + --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload. - --loop [auto|asyncio|uvloop] - Event loop implementation. [default: auto] + + --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] + --ws [auto|none|websockets|wsproto] WebSocket protocol implementation. [default: auto] + --lifespan [auto|on|off] Lifespan implementation. [default: auto] --interface [auto|asgi3|asgi2|wsgi] Select ASGI3, ASGI2, or WSGI as the application interface. [default: auto] + --env-file PATH Environment configuration file. - --log-config PATH Logging configuration file. - Supported formats (.ini, .json, .yaml) + --log-config PATH Logging configuration file. Supported + formats: .ini, .json, .yaml. + --log-level [critical|error|warning|info|debug|trace] Log level. [default: info] --access-log / --no-access-log Enable/Disable access log. @@ -61,37 +72,53 @@ Options: Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info. + --forwarded-allow-ips TEXT Comma seperated list of IPs to trust with proxy headers. Defaults to the $FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'. + --root-path TEXT Set the ASGI 'root_path' for applications submounted below a given URL path. + --limit-concurrency INTEGER Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses. + --backlog INTEGER Maximum number of connections to hold in backlog + --limit-max-requests INTEGER Maximum number of requests to service before terminating the process. + --timeout-keep-alive INTEGER Close Keep-Alive connections if no new data is received within this timeout. [default: 5] + --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file - --ssl-keyfile-password TEXT SSL key file password + --ssl-keyfile-password TEXT SSL keyfile password --ssl-version INTEGER SSL version to use (see stdlib ssl module's) [default: 2] + --ssl-cert-reqs INTEGER Whether client certificate is required (see stdlib ssl module's) [default: 0] + --ssl-ca-certs TEXT CA certificates file --ssl-ciphers TEXT Ciphers to use (see stdlib ssl module's) [default: TLSv1] + --header TEXT Specify custom default HTTP response headers as a Name:Value pair + + --version Display the uvicorn version and exit. --app-dir TEXT Look for APP in the specified directory, by adding this to the PYTHONPATH. Defaults to - the current working directory. + the current working directory. [default: .] + + --factory Treat APP as an application factory, i.e. a + () -> callable. [default: False] + --help Show this message and exit. ``` diff --git a/docs/index.md b/docs/index.md index a91d59e59..4e82ddf8f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -93,35 +93,47 @@ The uvicorn command line tool is the easiest way to run your application... ### Command line options + ``` +$ uvicorn --help Usage: uvicorn [OPTIONS] APP Options: --host TEXT Bind socket to this host. [default: 127.0.0.1] + --port INTEGER Bind socket to this port. [default: 8000] --uds TEXT Bind to a UNIX domain socket. --fd INTEGER Bind to socket from this file descriptor. --reload Enable auto-reload. --reload-dir TEXT Set reload directories explicitly, instead of using the current working directory. + + --reload-delay FLOAT Delay between previous and next check if + application needs to be. Defaults to 0.25s. + [default: 0.25] + --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if available. Not valid with --reload. - --loop [auto|asyncio|uvloop] - Event loop implementation. [default: auto] + + --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] + --ws [auto|none|websockets|wsproto] WebSocket protocol implementation. [default: auto] + --lifespan [auto|on|off] Lifespan implementation. [default: auto] --interface [auto|asgi3|asgi2|wsgi] Select ASGI3, ASGI2, or WSGI as the application interface. [default: auto] + --env-file PATH Environment configuration file. - --log-config PATH Logging configuration file. - Supported formats (.ini, .json, .yaml) + --log-config PATH Logging configuration file. Supported + formats: .ini, .json, .yaml. + --log-level [critical|error|warning|info|debug|trace] Log level. [default: info] --access-log / --no-access-log Enable/Disable access log. @@ -130,36 +142,53 @@ Options: Enable/Disable X-Forwarded-Proto, X-Forwarded-For, X-Forwarded-Port to populate remote address info. - --forwarded-allow-ips TEXT Comma separated list of IPs to trust with + + --forwarded-allow-ips TEXT Comma seperated list of IPs to trust with proxy headers. Defaults to the $FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'. + --root-path TEXT Set the ASGI 'root_path' for applications submounted below a given URL path. + --limit-concurrency INTEGER Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses. + --backlog INTEGER Maximum number of connections to hold in backlog + --limit-max-requests INTEGER Maximum number of requests to service before terminating the process. + --timeout-keep-alive INTEGER Close Keep-Alive connections if no new data is received within this timeout. [default: 5] + --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file - --ssl-keyfile-password TEXT SSL key file password + --ssl-keyfile-password TEXT SSL keyfile password --ssl-version INTEGER SSL version to use (see stdlib ssl module's) [default: 2] + --ssl-cert-reqs INTEGER Whether client certificate is required (see stdlib ssl module's) [default: 0] + --ssl-ca-certs TEXT CA certificates file --ssl-ciphers TEXT Ciphers to use (see stdlib ssl module's) [default: TLSv1] + --header TEXT Specify custom default HTTP response headers as a Name:Value pair + + --version Display the uvicorn version and exit. + --app-dir TEXT Look for APP in the specified directory, by + adding this to the PYTHONPATH. Defaults to + the current working directory. [default: .] + --factory Treat APP as an application factory, i.e. a - () -> function. [default: False] + () -> callable. [default: False] + --help Show this message and exit. ``` diff --git a/scripts/check b/scripts/check index 1f2190183..223c46d90 100755 --- a/scripts/check +++ b/scripts/check @@ -11,4 +11,5 @@ set -x ${PREFIX}black --check --diff --target-version=py36 $SOURCE_FILES ${PREFIX}flake8 $SOURCE_FILES ${PREFIX}mypy -${PREFIX}isort --check --diff --project=uvicorn $SOURCE_FILES \ No newline at end of file +${PREFIX}isort --check --diff --project=uvicorn $SOURCE_FILES +${PREFIX}python -m tools.cli_usage --check diff --git a/scripts/lint b/scripts/lint index 795d5d01a..ef06f6bdd 100755 --- a/scripts/lint +++ b/scripts/lint @@ -11,3 +11,4 @@ set -x ${PREFIX}autoflake --in-place --recursive $SOURCE_FILES ${PREFIX}isort --project=uvicorn $SOURCE_FILES ${PREFIX}black --target-version=py36 $SOURCE_FILES +${PREFIX}python -m tools.cli_usage diff --git a/tools/cli_usage.py b/tools/cli_usage.py new file mode 100644 index 000000000..09ed69a6c --- /dev/null +++ b/tools/cli_usage.py @@ -0,0 +1,70 @@ +""" +Look for a marker comment in docs pages, and place the output of +`$ uvicorn --help` there. Pass `--check` to ensure the content is in sync. +""" +import argparse +import subprocess +import sys +import typing +from pathlib import Path + + +def _get_usage_lines() -> typing.List[str]: + res = subprocess.run(["uvicorn", "--help"], stdout=subprocess.PIPE) + help_text = res.stdout.decode("utf-8") + return ["```", "$ uvicorn --help", *help_text.splitlines(), "```"] + + +def _find_next_codefence_lineno(lines: typing.List[str], after: int) -> int: + return next( + lineno for lineno, line in enumerate(lines[after:], after) if line == "```" + ) + + +def _get_insert_location(lines: typing.List[str]) -> typing.Tuple[int, int]: + marker = lines.index("") + start = marker + 1 + + if lines[start] == "```": + # Already generated. + # + # ``` <- start + # [...] + # ``` <- end + next_codefence = _find_next_codefence_lineno(lines, after=start + 1) + end = next_codefence + 1 + else: + # Not generated yet. + end = start + + return start, end + + +def _generate_cli_usage(path: Path, check: bool = False) -> int: + content = path.read_text() + + lines = content.splitlines() + usage_lines = _get_usage_lines() + start, end = _get_insert_location(lines) + lines = lines[:start] + usage_lines + lines[end:] + output = "\n".join(lines) + "\n" + + if check: + if content == output: + return 0 + print(f"ERROR: CLI usage in {path} is out of sync. Run scripts/lint to fix.") + return 1 + + path.write_text(output) + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + paths = [Path("docs", "index.md"), Path("docs", "deployment.md")] + rv = 0 + for path in paths: + rv |= _generate_cli_usage(path, check=args.check) + sys.exit(rv) diff --git a/uvicorn/main.py b/uvicorn/main.py index f82b98c0f..aba7e4af0 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -138,7 +138,7 @@ def print_version(ctx, param, value): "--log-config", type=click.Path(exists=True), default=None, - help="Logging configuration file.", + help="Logging configuration file. Supported formats: .ini, .json, .yaml.", show_default=True, ) @click.option( From 0660e7482d3592b8907e9ee61fce3f1ca6b888c7 Mon Sep 17 00:00:00 2001 From: euri10 Date: Wed, 9 Dec 2020 11:52:23 +0100 Subject: [PATCH 081/128] Removed extra kwrgs from access logging to avoid scope > headers leak (#859) --- uvicorn/protocols/http/h11_impl.py | 1 - uvicorn/protocols/http/httptools_impl.py | 1 - 2 files changed, 2 deletions(-) diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index e9583d70b..470a63077 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -460,7 +460,6 @@ async def send(self, message): get_path_with_query_string(self.scope), self.scope["http_version"], status_code, - extra={"status_code": status_code, "scope": self.scope}, ) # Write response status line and headers diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index 924e131ce..e60277422 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -462,7 +462,6 @@ async def send(self, message): get_path_with_query_string(self.scope), self.scope["http_version"], status_code, - extra={"status_code": status_code, "scope": self.scope}, ) # Write response status line and headers From 8581b348480c0f76a152d97de5f0304a84207341 Mon Sep 17 00:00:00 2001 From: Adam Hooper Date: Wed, 9 Dec 2020 12:06:14 -0500 Subject: [PATCH 082/128] Fix recv() errors when closed during handshake (#704) [closes #244] Previously, if the application tried to close the connection before the handshake completed, uvicorn would log: TypeError: An asyncio.Future, a coroutine or an awaitable is required. These TypeErrors happen in tests/protocols/test_websocket.py. pytest is hiding them. In my initial pull request, I added `caplog` to `test_websocket.py` to expose the bug: def test_close_connection(protocol_cls, caplog): caplog.set_level(logging.ERROR) # ... assert not [r.message for r in caplog.records] ... but this caused unrelated errors on Windows; so @florimondmanca suggests leaving the test suite alone, in https://github.com/encode/uvicorn/pull/704#issuecomment-740926960 --- uvicorn/protocols/websockets/websockets_impl.py | 9 +++++++++ uvicorn/protocols/websockets/wsproto_impl.py | 4 +--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index 407674cf0..dc681d2b5 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -216,6 +216,7 @@ async def asgi_send(self, message): elif message_type == "websocket.close": code = message.get("code", 1000) + self.close_code = code # for WebSocketServerProtocol await self.close(code) self.closed_event.set() @@ -236,6 +237,14 @@ async def asgi_receive(self): return {"type": "websocket.connect"} await self.handshake_completed_event.wait() + + if self.closed_event.is_set(): + # If the client disconnected: WebSocketServerProtocol set self.close_code. + # If the handshake failed or the app closed before handshake completion, + # use 1006 Abnormal Closure. + code = getattr(self, "close_code", 1006) + return {"type": "websocket.disconnect", "code": code} + try: data = await self.recv() except websockets.ConnectionClosed as exc: diff --git a/uvicorn/protocols/websockets/wsproto_impl.py b/uvicorn/protocols/websockets/wsproto_impl.py index d712963bd..456af1370 100644 --- a/uvicorn/protocols/websockets/wsproto_impl.py +++ b/uvicorn/protocols/websockets/wsproto_impl.py @@ -255,10 +255,8 @@ async def send(self, message): ) self.handshake_complete = True self.close_sent = True - msg = h11.Response(status_code=403, headers=[]) + msg = events.RejectConnection(status_code=403, headers=[]) output = self.conn.send(msg) - msg = h11.EndOfMessage() - output += self.conn.send(msg) self.transport.write(output) self.transport.close() From 9a3040c9cd56844631b28631acd5862b5a4eafdd Mon Sep 17 00:00:00 2001 From: euri10 Date: Wed, 9 Dec 2020 20:42:18 +0100 Subject: [PATCH 083/128] Ensure that ws connection is open before receiving data (#881) * Revert "Fix recv() errors when closed during handshake (#704)" This reverts commit 8581b348 * Fix 204 by using ensure_open that raises the disconnect --- uvicorn/protocols/websockets/websockets_impl.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index dc681d2b5..30be0dc2f 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -216,7 +216,6 @@ async def asgi_send(self, message): elif message_type == "websocket.close": code = message.get("code", 1000) - self.close_code = code # for WebSocketServerProtocol await self.close(code) self.closed_event.set() @@ -237,15 +236,8 @@ async def asgi_receive(self): return {"type": "websocket.connect"} await self.handshake_completed_event.wait() - - if self.closed_event.is_set(): - # If the client disconnected: WebSocketServerProtocol set self.close_code. - # If the handshake failed or the app closed before handshake completion, - # use 1006 Abnormal Closure. - code = getattr(self, "close_code", 1006) - return {"type": "websocket.disconnect", "code": code} - try: + await self.ensure_open() data = await self.recv() except websockets.ConnectionClosed as exc: return {"type": "websocket.disconnect", "code": exc.code} From a227f179533e6f38aef5897941d12506d6cb8ae6 Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 12 Dec 2020 11:55:18 +0100 Subject: [PATCH 084/128] Fix logger after scope removal (#884) * Fix logger after scope removal * Extra is not needed --- uvicorn/logging.py | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/uvicorn/logging.py b/uvicorn/logging.py index 1efaf92ba..5338329ad 100644 --- a/uvicorn/logging.py +++ b/uvicorn/logging.py @@ -1,7 +1,6 @@ import http import logging import sys -import urllib from copy import copy import click @@ -73,30 +72,12 @@ class AccessFormatter(ColourizedFormatter): 5: lambda code: click.style(str(code), fg="bright_red"), } - def get_client_addr(self, scope): - client = scope.get("client") - if not client: - return "" - return "%s:%d" % (client[0], client[1]) - - def get_path(self, scope): - return urllib.parse.quote(scope.get("root_path", "") + scope["path"]) - - def get_full_path(self, scope): - path = scope.get("root_path", "") + scope["path"] - query_string = scope.get("query_string", b"").decode("ascii") - if query_string: - return urllib.parse.quote(path) + "?" + query_string - return urllib.parse.quote(path) - - def get_status_code(self, record): - status_code = record.__dict__["status_code"] + def get_status_code(self, status_code: int): try: status_phrase = http.HTTPStatus(status_code).phrase except ValueError: status_phrase = "" status_and_phrase = "%s %s" % (status_code, status_phrase) - if self.use_colors: def default(code): @@ -108,25 +89,22 @@ def default(code): def formatMessage(self, record): recordcopy = copy(record) - scope = recordcopy.__dict__["scope"] - method = scope["method"] - path = self.get_path(scope) - full_path = self.get_full_path(scope) - client_addr = self.get_client_addr(scope) - status_code = self.get_status_code(recordcopy) - http_version = scope["http_version"] + ( + client_addr, + method, + full_path, + http_version, + status_code, + ) = recordcopy.args + status_code = self.get_status_code(status_code) request_line = "%s %s HTTP/%s" % (method, full_path, http_version) if self.use_colors: request_line = click.style(request_line, bold=True) recordcopy.__dict__.update( { - "method": method, - "path": path, - "full_path": full_path, "client_addr": client_addr, "request_line": request_line, "status_code": status_code, - "http_version": http_version, } ) return super().formatMessage(recordcopy) From 1f282e0218d09dd8859f68c9456bde3e17744c27 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 12 Dec 2020 12:55:14 +0100 Subject: [PATCH 085/128] Version 0.13.1 (#880) * Version 0.13.1 * Add #859 * Apply suggestions from code review Co-authored-by: euri10 Co-authored-by: euri10 --- CHANGELOG.md | 7 +++++++ uvicorn/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed58460e..47bd440c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Change Log +## 0.13.1 - 2020-12-12 + +### Fixed + +- Prevent exceptions when the ASGI application rejects a connection during the WebSocket handshake, when running on both `--ws wsproto` or `--ws websockets`. (Pull #704 and #881) +- Ensure connection `scope` doesn't leak in logs when using JSON log formatters. (Pull #859 and #884) + ## 0.13.0 - 2020-12-08 ### Added diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 184b5a329..ea3f0cc2f 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.13.0" +__version__ = "0.13.1" __all__ = ["main", "run", "Config", "Server"] From 305ed0e29261565ba6971a7bb75f4937f0d86957 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 12 Dec 2020 13:24:53 +0100 Subject: [PATCH 086/128] Log exception traceback in case of invalid HTTP request when using httptools (#886) --- uvicorn/protocols/http/httptools_impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index e60277422..7be086562 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -163,9 +163,9 @@ def data_received(self, data): try: self.parser.feed_data(data) - except httptools.HttpParserError: + except httptools.HttpParserError as exc: msg = "Invalid HTTP request received." - self.logger.warning(msg) + self.logger.warning(msg, exc_info=exc) self.transport.close() except httptools.HttpParserUpgrade: self.handle_upgrade() From 55129656c7fd2a2745f2532bf7262dbd0ed84bc6 Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 12 Dec 2020 14:20:58 +0100 Subject: [PATCH 087/128] Increase coverage by testing for an invalid http request (#857) * Increase coverage by testing for an invalid http request * Tweak buffer check Co-authored-by: Florimond Manca --- tests/protocols/test_http.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/protocols/test_http.py b/tests/protocols/test_http.py index 298714850..361264a20 100644 --- a/tests/protocols/test_http.py +++ b/tests/protocols/test_http.py @@ -72,6 +72,10 @@ ] ) +INVALID_REQUEST = b"\r\n".join( + [b"GET /?x=y z HTTP/1.1", b"Host: example.org", b"", b""] # bad space character +) + class MockTransport: def __init__(self, sockname=None, peername=None, sslcontext=False): @@ -700,3 +704,17 @@ def test_scopes(asgi2or3_app, expected_scopes, protocol_cls): protocol.data_received(SIMPLE_GET_REQUEST) protocol.loop.run_one() assert expected_scopes == protocol.scope.get("asgi") + + +@pytest.mark.parametrize("protocol_cls", HTTP_PROTOCOLS) +def test_invalid_http_request(protocol_cls, caplog): + def app(scope): + return Response("Hello, world", media_type="text/plain") + + caplog.set_level(logging.INFO, logger="uvicorn.error") + logging.getLogger("uvicorn.error").propagate = True + + protocol = get_connected_protocol(app, protocol_cls) + protocol.data_received(INVALID_REQUEST) + assert not protocol.transport.buffer + assert "Invalid HTTP request received." in caplog.messages From 2e17dd9ff3cbb826237639005fe2cd24bc8d225c Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 12 Dec 2020 18:32:45 +0100 Subject: [PATCH 088/128] Log exception traceback in case of invalid HTTP request when using h11 (#889) --- uvicorn/protocols/http/h11_impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index 470a63077..5d2bad4a1 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -168,9 +168,9 @@ def handle_events(self): while True: try: event = self.conn.next_event() - except h11.RemoteProtocolError: + except h11.RemoteProtocolError as exc: msg = "Invalid HTTP request received." - self.logger.warning(msg) + self.logger.warning(msg, exc_info=exc) self.transport.close() return event_type = type(event) From 5e8a88849dbce5be554110113ad7ebdbba87f1ac Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 12 Dec 2020 18:33:10 +0100 Subject: [PATCH 089/128] Add extra invalid HTTP request test cases (#888) --- tests/protocols/test_http.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/protocols/test_http.py b/tests/protocols/test_http.py index 361264a20..f35a965d6 100644 --- a/tests/protocols/test_http.py +++ b/tests/protocols/test_http.py @@ -72,8 +72,13 @@ ] ) -INVALID_REQUEST = b"\r\n".join( - [b"GET /?x=y z HTTP/1.1", b"Host: example.org", b"", b""] # bad space character +INVALID_REQUEST_TEMPLATE = b"\r\n".join( + [ + b"%s", + b"Host: example.org", + b"", + b"", + ] ) @@ -706,15 +711,23 @@ def test_scopes(asgi2or3_app, expected_scopes, protocol_cls): assert expected_scopes == protocol.scope.get("asgi") +@pytest.mark.parametrize( + "request_line", + [ + pytest.param(b"G?T / HTTP/1.1", id="invalid-method"), + pytest.param(b"GET /?x=y z HTTP/1.1", id="invalid-path"), + pytest.param(b"GET / HTTP1.1", id="invalid-http-version"), + ], +) @pytest.mark.parametrize("protocol_cls", HTTP_PROTOCOLS) -def test_invalid_http_request(protocol_cls, caplog): - def app(scope): - return Response("Hello, world", media_type="text/plain") +def test_invalid_http_request(request_line, protocol_cls, caplog): + app = Response("Hello, world", media_type="text/plain") + request = INVALID_REQUEST_TEMPLATE % request_line caplog.set_level(logging.INFO, logger="uvicorn.error") logging.getLogger("uvicorn.error").propagate = True protocol = get_connected_protocol(app, protocol_cls) - protocol.data_received(INVALID_REQUEST) + protocol.data_received(request) assert not protocol.transport.buffer assert "Invalid HTTP request received." in caplog.messages From 058234ffac3a5c50828b254e1750a4252286733a Mon Sep 17 00:00:00 2001 From: Roberto Prevato Date: Sun, 13 Dec 2020 10:53:56 +0100 Subject: [PATCH 090/128] adds BlackSheep to the list of ASGI web frameworks (#891) --- docs/index.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/index.md b/docs/index.md index 4e82ddf8f..be1a096ca 100644 --- a/docs/index.md +++ b/docs/index.md @@ -481,6 +481,12 @@ with application code still running in a standard threaded context. You write your API function parameters with Python 3.6+ type declarations and get automatic data conversion, data validation, OpenAPI schemas (with JSON Schemas) and interactive API documentation UIs. +### BlackSheep + +[BlackSheep](https://www.neoteroi.dev/blacksheep/) is a web framework based on ASGI, inspired by Flask and ASP.NET Core. + +Its most distinctive features are built-in support for dependency injection, automatic binding of parameters by request handler's type annotations, and automatic generation of OpenAPI documentation and Swagger UI. + [uvloop]: https://github.com/MagicStack/uvloop [httptools]: https://github.com/MagicStack/httptools From debdb64c71fd09aa6e89652a4906d10cac31551b Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 17 Dec 2020 18:00:07 +0100 Subject: [PATCH 091/128] Brings lifespan to 100% coverage (#897) --- tests/test_lifespan.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py index d72269ab1..e3413affc 100644 --- a/tests/test_lifespan.py +++ b/tests/test_lifespan.py @@ -123,11 +123,13 @@ async def test(): @pytest.mark.parametrize("mode", ("auto", "on")) @pytest.mark.parametrize("raise_exception", (True, False)) -def test_lifespan_with_failed_startup(mode, raise_exception): +def test_lifespan_with_failed_startup(mode, raise_exception, caplog): async def app(scope, receive, send): message = await receive() assert message["type"] == "lifespan.startup" - await send({"type": "lifespan.startup.failed"}) + await send( + {"type": "lifespan.startup.failed", "message": "the lifespan event failed"} + ) if raise_exception: # App should be able to re-raise an exception if startup failed. raise RuntimeError() @@ -144,6 +146,13 @@ async def test(): loop = asyncio.new_event_loop() loop.run_until_complete(test()) + error_messages = [ + record.message + for record in caplog.records + if record.name == "uvicorn.error" and record.levelname == "ERROR" + ] + assert "the lifespan event failed" in error_messages.pop(0) + assert "Application startup failed. Exiting." in error_messages.pop(0) @pytest.mark.parametrize("mode", ("auto", "on")) From 1841655a981f5b3d78b7cebc7ad58c4f17f44dd2 Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 19 Dec 2020 10:55:16 +0100 Subject: [PATCH 092/128] Config coverage upgrade (#898) * Small test config up * Correct indent --- tests/test_config.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 67126f087..ae8e987e5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -65,12 +65,27 @@ def test_proxy_headers(): assert isinstance(config.loaded_app, ProxyHeadersMiddleware) -def test_app_unimportable(): +def test_app_unimportable_module(): config = Config(app="no.such:app") with pytest.raises(ImportError): config.load() +def test_app_unimportable_other(caplog): + config = Config(app="tests.test_config:app") + with pytest.raises(SystemExit): + config.load() + error_messages = [ + record.message + for record in caplog.records + if record.name == "uvicorn.error" and record.levelname == "ERROR" + ] + assert ( + 'Error loading ASGI app. Attribute "app" not found in module "tests.test_config".' # noqa: E501 + == error_messages.pop(0) + ) + + def test_app_factory(): def create_app(): return asgi_app From cdb687335dc5321485e3dc866c8495e2e3df243e Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 19 Dec 2020 12:45:38 +0100 Subject: [PATCH 093/128] Test config should_reload property (#899) * Test config should_reload property * Separate test --- tests/test_config.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index ae8e987e5..fdbf275a4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -48,6 +48,20 @@ def test_debug_app(): assert isinstance(config.loaded_app, DebugMiddleware) +@pytest.mark.parametrize( + "app, expected_should_reload", + [(asgi_app, False), ("tests.test_config:asgi_app", True)], +) +def test_config_should_reload_is_set(app, expected_should_reload): + config_debug = Config(app=app, debug=True) + assert config_debug.debug is True + assert config_debug.should_reload is expected_should_reload + + config_reload = Config(app=app, reload=True) + assert config_reload.reload is True + assert config_reload.should_reload is expected_should_reload + + def test_wsgi_app(): config = Config(app=wsgi_app, interface="wsgi", proxy_headers=False) config.load() From 73d6f096869cc3bf698008e0d2b5763e09699a43 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 19 Dec 2020 20:53:05 +0100 Subject: [PATCH 094/128] Release 0.13.2 (#890) --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47bd440c4..db0157c84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## 0.13.2 - 2020-12-12 + +### Fixed + +- Log full exception traceback in case of invalid HTTP request. (Pull #886 and #888) + ## 0.13.1 - 2020-12-12 ### Fixed From 7a667a582baf374ad40775af77ce6271b473d0f0 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sun, 20 Dec 2020 09:40:05 +0100 Subject: [PATCH 095/128] Bump version in __init__.py (#905) --- uvicorn/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index ea3f0cc2f..6a72df40d 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.13.1" +__version__ = "0.13.2" __all__ = ["main", "run", "Config", "Server"] From e01c927d76bece04a9fdbdf530ca8abeb7f2b557 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 22 Dec 2020 14:41:57 +0100 Subject: [PATCH 096/128] Small coverage increase in protocols/utils (#904) --- tests/protocols/test_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/protocols/test_utils.py b/tests/protocols/test_utils.py index 0fff34a5d..01fe0ddbd 100644 --- a/tests/protocols/test_utils.py +++ b/tests/protocols/test_utils.py @@ -1,6 +1,8 @@ import socket -from uvicorn.protocols.utils import get_local_addr, get_remote_addr +import pytest + +from uvicorn.protocols.utils import get_client_addr, get_local_addr, get_remote_addr class MockSocket: @@ -80,3 +82,12 @@ def test_get_remote_addr(): transport = MockTransport({"peername": ("123.45.6.7", 123)}) assert get_remote_addr(transport) == ("123.45.6.7", 123) + + +@pytest.mark.parametrize( + "scope, expected_client", + [({"client": ("127.0.0.1", 36000)}, "127.0.0.1:36000"), ({"client": None}, "")], + ids=["ip:port client", "None client"], +) +def test_get_client_addr(scope, expected_client): + assert get_client_addr(scope) == expected_client From 39b3903c5c675db1723334f7548ed0524943ce47 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 22 Dec 2020 16:01:53 +0100 Subject: [PATCH 097/128] Fix docs about reloader (#907) --- docs/settings.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index a3dad5c8c..2a717ab93 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -22,12 +22,6 @@ equivalent keyword arguments, eg. `uvicorn.run("example:app", port=5000, reload= * `--reload` - Enable auto-reload. * `--reload-dir ` - Specify which directories to watch for python file changes. May be used multiple times. If unused, then by default all directories in current directory will be watched. -By default Uvicorn uses simple changes detection strategy that compares python files modification times few times a second. If this approach doesn't work for your project (eg. because of its complexity), you can install Uvicorn with optional `watchgod` dependency to use filesystem events instead: - -``` -$ pip install uvicorn[watchgodreload] -``` - ## Production * `--workers ` - Use multiple worker processes. Defaults to the value of the `$WEB_CONCURRENCY` environment variable. From 49c0a5525cbf51a98225ca3f9273f376aed024b9 Mon Sep 17 00:00:00 2001 From: euri10 Date: Wed, 23 Dec 2020 10:09:09 +0100 Subject: [PATCH 098/128] Added docs example to tweak gunicorn worker config (#908) --- docs/deployment.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/deployment.md b/docs/deployment.md index 4ace0b73d..cec0d8ef8 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -176,6 +176,13 @@ The `UvicornWorker` implementation uses the `uvloop` and `httptools` implementat Gunicorn provides a different set of configuration options to Uvicorn, so some options such as `--limit-concurrency` are not yet supported when running with Gunicorn. +If you need to pass uvicorn's config arguments to gunicorn workers then you'll have to subclass `UvicornWorker`: + +```python +class MyUvicornWorker(UvicornWorker): + CONFIG_KWARGS = {"loop": "asyncio", "http": "h11", "lifespan": "off"} +``` + ### Supervisor To use `supervisor` as a process manager you should either: From f8dd9fb41b0385166e3127fd68f9b73de4f28bec Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 26 Dec 2020 16:34:14 +0100 Subject: [PATCH 099/128] Fix lifespan tests (#913) --- tests/test_lifespan.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py index e3413affc..de584e1a8 100644 --- a/tests/test_lifespan.py +++ b/tests/test_lifespan.py @@ -155,26 +155,33 @@ async def test(): assert "Application startup failed. Exiting." in error_messages.pop(0) -@pytest.mark.parametrize("mode", ("auto", "on")) -def test_lifespan_scope_asgi3app(mode): +def test_lifespan_scope_asgi3app(): async def asgi3app(scope, receive, send): - assert scope == {"version": "3.0", "spec_version": "2.0"} + assert scope == { + "type": "lifespan", + "asgi": {"version": "3.0", "spec_version": "2.0"}, + } async def test(): - config = Config(app=asgi3app, lifespan=mode) + config = Config(app=asgi3app, lifespan="on") lifespan = LifespanOn(config) await lifespan.startup() + assert not lifespan.startup_failed + assert not lifespan.error_occured + assert not lifespan.should_exit await lifespan.shutdown() loop = asyncio.new_event_loop() loop.run_until_complete(test()) -@pytest.mark.parametrize("mode", ("auto", "on")) -def test_lifespan_scope_asgi2app(mode): +def test_lifespan_scope_asgi2app(): def asgi2app(scope): - assert scope == {"version": "2.0", "spec_version": "2.0"} + assert scope == { + "type": "lifespan", + "asgi": {"version": "2.0", "spec_version": "2.0"}, + } async def asgi(receive, send): pass @@ -182,7 +189,7 @@ async def asgi(receive, send): return asgi async def test(): - config = Config(app=asgi2app, lifespan=mode) + config = Config(app=asgi2app, lifespan="on") lifespan = LifespanOn(config) await lifespan.startup() From afb2d565c8dae859bcef4c76b3c6dc3f3077314d Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sat, 26 Dec 2020 16:36:35 +0100 Subject: [PATCH 100/128] Tweak detection of app factories (#914) --- tests/test_config.py | 14 ++++++++++---- uvicorn/config.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index fdbf275a4..773be2113 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,5 @@ import json +import logging import socket from copy import deepcopy @@ -100,7 +101,7 @@ def test_app_unimportable_other(caplog): ) -def test_app_factory(): +def test_app_factory(caplog): def create_app(): return asgi_app @@ -108,10 +109,15 @@ def create_app(): config.load() assert config.loaded_app is asgi_app - # Flag missing. - config = Config(app=create_app) - with pytest.raises(SystemExit): + # Flag not passed. In this case, successfully load the app, but issue a warning + # to indicate that an explicit flag is preferred. + caplog.clear() + config = Config(app=create_app, proxy_headers=False) + with caplog.at_level(logging.WARNING): config.load() + assert config.loaded_app is asgi_app + assert len(caplog.records) == 1 + assert "--factory" in caplog.records[0].message # App not a no-arguments callable. config = Config(app=asgi_app, factory=True) diff --git a/uvicorn/config.py b/uvicorn/config.py index 62fe3c5d0..6cd4ced7a 100644 --- a/uvicorn/config.py +++ b/uvicorn/config.py @@ -310,18 +310,18 @@ def load(self): logger.error("Error loading ASGI app. %s" % exc) sys.exit(1) - if self.factory: - try: - self.loaded_app = self.loaded_app() - except TypeError as exc: + try: + self.loaded_app = self.loaded_app() + except TypeError as exc: + if self.factory: logger.error("Error loading ASGI app factory: %s", exc) sys.exit(1) - elif not inspect.signature(self.loaded_app).parameters: - logger.error( - "APP seems to be an application factory. " - "Run uvicorn with the --factory flag." - ) - sys.exit(1) + else: + if not self.factory: + logger.warning( + "ASGI app factory detected. Using it, " + "but please consider setting the --factory flag explicitly." + ) if self.interface == "auto": if inspect.isclass(self.loaded_app): From 28f5d2bede7e6d70f191c8617680e5835b94e34c Mon Sep 17 00:00:00 2001 From: Matthew Clapp Date: Sun, 27 Dec 2020 08:08:24 -0800 Subject: [PATCH 101/128] Make setup.py PEP440 compliant. (#916) --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ab7a751da..61ea105e8 100755 --- a/setup.py +++ b/setup.py @@ -53,9 +53,9 @@ def get_packages(package): "websockets==8.*", "httptools==0.1.* ;" + env_marker_cpython, "uvloop>=0.14.0 ;" + env_marker_cpython, - "colorama>=0.4.*;" + env_marker_win, + "colorama>=0.4;" + env_marker_win, "watchgod>=0.6,<0.7", - "python-dotenv>=0.13.*", + "python-dotenv>=0.13", "PyYAML>=5.1", ] From c6b80fce49e9ad53f514262f4c1eb7f9a6798620 Mon Sep 17 00:00:00 2001 From: SergBobrovsky Date: Sun, 27 Dec 2020 22:45:37 +0300 Subject: [PATCH 102/128] remove redundant transform list -> list (#909) --- uvicorn/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvicorn/main.py b/uvicorn/main.py index aba7e4af0..6bfc631f0 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -355,7 +355,7 @@ def main( "ssl_cert_reqs": ssl_cert_reqs, "ssl_ca_certs": ssl_ca_certs, "ssl_ciphers": ssl_ciphers, - "headers": list([header.split(":", 1) for header in headers]), + "headers": [header.split(":", 1) for header in headers], "use_colors": use_colors, "factory": factory, } From 70ec20fea81b9520e9227beea699d5822cf888b2 Mon Sep 17 00:00:00 2001 From: euri10 Date: Mon, 28 Dec 2020 09:50:06 +0100 Subject: [PATCH 103/128] Test suite refactor using async server context manager instead of background thread (#917) * Used async context manager instead of thread * Refactored test_ssl * Refactored test_main , last test become irrelevant * Same with test_websockets, now exceptions are bubbled ! * Update requirements.txt * Refactor test_trace_logging.py as well * Removed what appears to be dead code since we dont use threads * asynccontextmanager appears in 3.7, just removing it to check if CI passes * trace logging reworked * Attempt at 3.6 * Should be better with correct lib name * Pin new requirements * Move run_server from conftest to utils * Revert "Removed what appears to be dead code since we dont use threads" This reverts commit eb45afc1 --- requirements.txt | 4 + tests/middleware/test_trace_logging.py | 47 ++--- tests/protocols/test_websocket.py | 228 +++++++++++-------------- tests/test_default_headers.py | 93 ++++------ tests/test_main.py | 79 ++------- tests/test_ssl.py | 75 ++------ tests/utils.py | 20 +++ 7 files changed, 207 insertions(+), 339 deletions(-) create mode 100644 tests/utils.py diff --git a/requirements.txt b/requirements.txt index 7fde8b844..ae07f9654 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,10 @@ mypy trustme cryptography coverage +httpx==0.16.* +pytest-asyncio==0.14.* +async_generator; python_version < '3.7' + # Documentation mkdocs diff --git a/tests/middleware/test_trace_logging.py b/tests/middleware/test_trace_logging.py index ce3c37f27..314e7eda0 100644 --- a/tests/middleware/test_trace_logging.py +++ b/tests/middleware/test_trace_logging.py @@ -1,12 +1,8 @@ -import platform -import sys -import threading -import time - +import httpx import pytest -import requests -from uvicorn import Config, Server +from tests.utils import run_server +from uvicorn import Config test_logging_config = { "version": 1, @@ -54,11 +50,8 @@ async def app(scope, receive, send): await send({"type": "http.response.body", "body": b"", "more_body": False}) -@pytest.mark.skipif( - sys.platform.startswith("win") or platform.python_implementation() == "PyPy", - reason="Skipping test on Windows and PyPy", -) -def test_trace_logging(capsys): +@pytest.mark.asyncio +async def test_trace_logging(capsys): config = Config( app=app, loop="asyncio", @@ -66,40 +59,28 @@ def test_trace_logging(capsys): log_config=test_logging_config, log_level="trace", ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") assert response.status_code == 204 - thread.join() captured = capsys.readouterr() assert '"GET / HTTP/1.1" 204' in captured.out assert "[TEST_ACCESS] TRACE" not in captured.out -@pytest.mark.skipif( - sys.platform.startswith("win") or platform.python_implementation() == "PyPy", - reason="Skipping test on Windows and PyPy", -) -@pytest.mark.parametrize("http_protocol", [("h11"), ("httptools")]) -def test_access_logging(capsys, http_protocol): +@pytest.mark.asyncio +async def test_access_logging(capsys): config = Config( app=app, loop="asyncio", - http=http_protocol, limit_max_requests=1, log_config=test_logging_config, ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") + assert response.status_code == 204 - thread.join() captured = capsys.readouterr() assert '"GET / HTTP/1.1" 204' in captured.out assert "uvicorn.access" in captured.out diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 71b81fcd2..24bc285ff 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -1,15 +1,8 @@ -import asyncio -import functools -import threading -import time -from contextlib import contextmanager - +import httpx import pytest -import requests +from tests.utils import run_server from uvicorn.config import Config -from uvicorn.main import ServerState -from uvicorn.protocols.http.h11_impl import H11Protocol from uvicorn.protocols.websockets.wsproto_impl import WSProtocol try: @@ -47,46 +40,20 @@ async def asgi(self): break -def run_loop(loop): - loop.run_forever() - loop.close() - - -@contextmanager -def run_server(app, protocol_cls, path="/"): - asyncio.set_event_loop(None) - loop = asyncio.new_event_loop() - config = Config(app=app, ws=protocol_cls) - server_state = ServerState() - protocol = functools.partial(H11Protocol, config=config, server_state=server_state) - create_server_task = loop.create_server(protocol, host="127.0.0.1") - server = loop.run_until_complete(create_server_task) - port = server.sockets[0].getsockname()[1] - url = "ws://127.0.0.1:{port}{path}".format(port=port, path=path) - try: - # Run the event loop in a new thread. - thread = threading.Thread(target=run_loop, args=[loop]) - thread.start() - # Return the contextmanager state. - yield url - finally: - # Close the loop from our main thread. - while server_state.tasks: - time.sleep(0.01) - loop.call_soon_threadsafe(loop.stop) - thread.join() - - +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_invalid_upgrade(protocol_cls): +async def test_invalid_upgrade(protocol_cls): def app(scope): return None - with run_server(app, protocol_cls=protocol_cls) as url: - url = url.replace("ws://", "http://") - response = requests.get( - url, headers={"upgrade": "websocket", "connection": "upgrade"}, timeout=5 - ) + config = Config(app=app, ws=protocol_cls) + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get( + "http://127.0.0.1:8000", + headers={"upgrade": "websocket", "connection": "upgrade"}, + timeout=5, + ) if response.status_code == 426: # response.text == "" pass # ok, wsproto 0.13 @@ -103,8 +70,9 @@ def app(scope): ] +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_accept_connection(protocol_cls): +async def test_accept_connection(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -113,15 +81,15 @@ async def open_connection(url): async with websockets.connect(url) as websocket: return websocket.open - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - is_open = loop.run_until_complete(open_connection(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + is_open = await open_connection("ws://127.0.0.1:8000") assert is_open - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_close_connection(protocol_cls): +async def test_close_connection(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.close"}) @@ -133,15 +101,15 @@ async def open_connection(url): return False return True # pragma: no cover - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - is_open = loop.run_until_complete(open_connection(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + is_open = await open_connection("ws://127.0.0.1:8000") assert not is_open - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_headers(protocol_cls): +async def test_headers(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): headers = self.scope.get("headers") @@ -153,15 +121,15 @@ async def open_connection(url): async with websockets.connect(url) as websocket: return websocket.open - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - is_open = loop.run_until_complete(open_connection(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + is_open = await open_connection("ws://127.0.0.1:8000") assert is_open - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_path_and_raw_path(protocol_cls): +async def test_path_and_raw_path(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): path = self.scope.get("path") @@ -174,14 +142,15 @@ async def open_connection(url): async with websockets.connect(url) as websocket: return websocket.open - with run_server(App, protocol_cls=protocol_cls, path="/one%2Ftwo") as url: - loop = asyncio.new_event_loop() - loop.run_until_complete(open_connection(url)) - loop.close() + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + is_open = await open_connection("ws://127.0.0.1:8000/one%2Ftwo") + assert is_open +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_text_data_to_client(protocol_cls): +async def test_send_text_data_to_client(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -191,15 +160,15 @@ async def get_data(url): async with websockets.connect(url) as websocket: return await websocket.recv() - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - data = loop.run_until_complete(get_data(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + data = await get_data("ws://127.0.0.1:8000") assert data == "123" - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_binary_data_to_client(protocol_cls): +async def test_send_binary_data_to_client(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -209,15 +178,15 @@ async def get_data(url): async with websockets.connect(url) as websocket: return await websocket.recv() - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - data = loop.run_until_complete(get_data(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + data = await get_data("ws://127.0.0.1:8000") assert data == b"123" - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_and_close_connection(protocol_cls): +async def test_send_and_close_connection(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -234,16 +203,16 @@ async def get_data(url): is_open = False return (data, is_open) - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - (data, is_open) = loop.run_until_complete(get_data(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + (data, is_open) = await get_data("ws://127.0.0.1:8000") assert data == "123" assert not is_open - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_text_data_to_server(protocol_cls): +async def test_send_text_data_to_server(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -257,15 +226,15 @@ async def send_text(url): await websocket.send("abc") return await websocket.recv() - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - data = loop.run_until_complete(send_text(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + data = await send_text("ws://127.0.0.1:8000") assert data == "abc" - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_binary_data_to_server(protocol_cls): +async def test_send_binary_data_to_server(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -279,15 +248,15 @@ async def send_text(url): await websocket.send(b"abc") return await websocket.recv() - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - data = loop.run_until_complete(send_text(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + data = await send_text("ws://127.0.0.1:8000") assert data == b"abc" - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_after_protocol_close(protocol_cls): +async def test_send_after_protocol_close(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -306,48 +275,48 @@ async def get_data(url): is_open = False return (data, is_open) - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - (data, is_open) = loop.run_until_complete(get_data(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + (data, is_open) = await get_data("ws://127.0.0.1:8000") assert data == "123" assert not is_open - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_missing_handshake(protocol_cls): +async def test_missing_handshake(protocol_cls): async def app(app, receive, send): pass async def connect(url): await websockets.connect(url) - with run_server(app, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() + config = Config(app=app, ws=protocol_cls, lifespan="off") + async with run_server(config): with pytest.raises(websockets.exceptions.InvalidStatusCode) as exc_info: - loop.run_until_complete(connect(url)) + await connect("ws://127.0.0.1:8000") assert exc_info.value.status_code == 500 - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_send_before_handshake(protocol_cls): +async def test_send_before_handshake(protocol_cls): async def app(scope, receive, send): await send({"type": "websocket.send", "text": "123"}) async def connect(url): await websockets.connect(url) - with run_server(app, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() + config = Config(app=app, ws=protocol_cls, lifespan="off") + async with run_server(config): with pytest.raises(websockets.exceptions.InvalidStatusCode) as exc_info: - loop.run_until_complete(connect(url)) + await connect("ws://127.0.0.1:8000") assert exc_info.value.status_code == 500 - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_duplicate_handshake(protocol_cls): +async def test_duplicate_handshake(protocol_cls): async def app(scope, receive, send): await send({"type": "websocket.accept"}) await send({"type": "websocket.accept"}) @@ -356,16 +325,16 @@ async def connect(url): async with websockets.connect(url) as websocket: _ = await websocket.recv() - with run_server(app, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() + config = Config(app=app, ws=protocol_cls, lifespan="off") + async with run_server(config): with pytest.raises(websockets.exceptions.ConnectionClosed) as exc_info: - loop.run_until_complete(connect(url)) + await connect("ws://127.0.0.1:8000") assert exc_info.value.code == 1006 - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_asgi_return_value(protocol_cls): +async def test_asgi_return_value(protocol_cls): """ The ASGI callable should return 'None'. If it doesn't make sure that the connection is closed with an error condition. @@ -379,16 +348,16 @@ async def connect(url): async with websockets.connect(url) as websocket: _ = await websocket.recv() - with run_server(app, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() + config = Config(app=app, ws=protocol_cls, lifespan="off") + async with run_server(config): with pytest.raises(websockets.exceptions.ConnectionClosed) as exc_info: - loop.run_until_complete(connect(url)) + await connect("ws://127.0.0.1:8000") assert exc_info.value.code == 1006 - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_app_close(protocol_cls): +async def test_app_close(protocol_cls): async def app(scope, receive, send): while True: message = await receive() @@ -405,16 +374,16 @@ async def websocket_session(url): await websocket.send("abc") await websocket.recv() - with run_server(app, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() + config = Config(app=app, ws=protocol_cls, lifespan="off") + async with run_server(config): with pytest.raises(websockets.exceptions.ConnectionClosed) as exc_info: - loop.run_until_complete(websocket_session(url)) + await websocket_session("ws://127.0.0.1:8000") assert exc_info.value.code == 1000 - loop.close() +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -def test_client_close(protocol_cls): +async def test_client_close(protocol_cls): async def app(scope, receive, send): while True: message = await receive() @@ -430,15 +399,15 @@ async def websocket_session(url): await websocket.ping() await websocket.send("abc") - with run_server(app, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - loop.run_until_complete(websocket_session(url)) - loop.close() + config = Config(app=app, ws=protocol_cls, lifespan="off") + async with run_server(config): + await websocket_session("ws://127.0.0.1:8000") +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) @pytest.mark.parametrize("subprotocol", ["proto1", "proto2"]) -def test_subprotocols(protocol_cls, subprotocol): +async def test_subprotocols(protocol_cls, subprotocol): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept", "subprotocol": subprotocol}) @@ -449,8 +418,7 @@ async def get_subprotocol(url): ) as websocket: return websocket.subprotocol - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - accepted_subprotocol = loop.run_until_complete(get_subprotocol(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + accepted_subprotocol = await get_subprotocol("ws://127.0.0.1:8000") assert accepted_subprotocol == subprotocol - loop.close() diff --git a/tests/test_default_headers.py b/tests/test_default_headers.py index 435d64b61..32000cb32 100644 --- a/tests/test_default_headers.py +++ b/tests/test_default_headers.py @@ -1,9 +1,8 @@ -import threading -import time +import httpx +import pytest -import requests - -from uvicorn import Config, Server +from tests.utils import run_server +from uvicorn import Config async def app(scope, receive, send): @@ -12,79 +11,61 @@ async def app(scope, receive, send): await send({"type": "http.response.body", "body": b"", "more_body": False}) -def test_default_default_headers(): +@pytest.mark.asyncio +async def test_default_default_headers(): config = Config(app=app, loop="asyncio", limit_max_requests=1) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") - - assert response.headers["server"] == "uvicorn" and response.headers["date"] - - thread.join() + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") + assert response.headers["server"] == "uvicorn" and response.headers["date"] -def test_override_server_header(): +@pytest.mark.asyncio +async def test_override_server_header(): config = Config( app=app, loop="asyncio", limit_max_requests=1, headers=[("Server", "over-ridden")], ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") + assert ( + response.headers["server"] == "over-ridden" and response.headers["date"] + ) - assert response.headers["server"] == "over-ridden" and response.headers["date"] - thread.join() - - -def test_override_server_header_multiple_times(): +@pytest.mark.asyncio +async def test_override_server_header_multiple_times(): config = Config( app=app, loop="asyncio", limit_max_requests=1, headers=[("Server", "over-ridden"), ("Server", "another-value")], ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") + assert ( + response.headers["server"] == "over-ridden, another-value" + and response.headers["date"] + ) - assert ( - response.headers["server"] == "over-ridden, another-value" - and response.headers["date"] - ) - thread.join() - - -def test_add_additional_header(): +@pytest.mark.asyncio +async def test_add_additional_header(): config = Config( app=app, loop="asyncio", limit_max_requests=1, headers=[("X-Additional", "new-value")], ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") - - assert ( - response.headers["x-additional"] == "new-value" - and response.headers["server"] == "uvicorn" - and response.headers["date"] - ) - - thread.join() + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") + assert ( + response.headers["x-additional"] == "new-value" + and response.headers["server"] == "uvicorn" + and response.headers["date"] + ) diff --git a/tests/test_main.py b/tests/test_main.py index 845a9c9a6..5a5f3fd88 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,12 +1,8 @@ -import asyncio -import threading -import time - +import httpx import pytest -import requests +from tests.utils import run_server from uvicorn.config import Config -from uvicorn.main import Server async def app(scope, receive, send): @@ -15,6 +11,7 @@ async def app(scope, receive, send): await send({"type": "http.response.body", "body": b"", "more_body": False}) +@pytest.mark.asyncio @pytest.mark.parametrize( "host, url", [ @@ -23,69 +20,27 @@ async def app(scope, receive, send): pytest.param("::1", "http://[::1]:8000", id="ipv6"), ], ) -def test_run(host, url): +async def test_run(host, url): config = Config(app=app, host=host, loop="asyncio", limit_max_requests=1) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get(url) + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get(url) assert response.status_code == 204 - thread.join() -def test_run_multiprocess(): +@pytest.mark.asyncio +async def test_run_multiprocess(): config = Config(app=app, loop="asyncio", workers=2, limit_max_requests=1) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") assert response.status_code == 204 - thread.join() -def test_run_reload(): +@pytest.mark.asyncio +async def test_run_reload(): config = Config(app=app, loop="asyncio", reload=True, limit_max_requests=1) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - response = requests.get("http://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get("http://127.0.0.1:8000") assert response.status_code == 204 - thread.join() - - -def test_run_with_shutdown(): - async def app(scope, receive, send): - assert scope["type"] == "http" - while True: - time.sleep(1) - - config = Config(app=app, loop="asyncio", workers=2, limit_max_requests=1) - server = Server(config=config) - sock = config.bind_socket() - exc = True - - def safe_run(): - nonlocal exc, server - try: - exc = None - config.setup_event_loop() - loop = asyncio.get_event_loop() - loop.run_until_complete(server.serve(sockets=[sock])) - except Exception as e: - exc = e - - thread = threading.Thread(target=safe_run) - thread.start() - - while not server.started: - time.sleep(0.01) - - server.should_exit = True - thread.join() - assert exc is None diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 19948ae4c..4cea0963b 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -1,28 +1,8 @@ -import contextlib -import sys -import threading -import time -import warnings -from functools import partialmethod - +import httpx import pytest -import requests -from urllib3.exceptions import InsecureRequestWarning +from tests.utils import run_server from uvicorn.config import Config -from uvicorn.main import Server - - -@contextlib.contextmanager -def no_ssl_verification(session=requests.Session): - old_request = session.request - session.request = partialmethod(old_request, verify=False) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", InsecureRequestWarning) - yield - - session.request = old_request async def app(scope, receive, send): @@ -31,10 +11,8 @@ async def app(scope, receive, send): await send({"type": "http.response.body", "body": b"", "more_body": False}) -@pytest.mark.skipif( - sys.platform.startswith("win"), reason="Skipping SSL test on Windows" -) -def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): +@pytest.mark.asyncio +async def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): config = Config( app=app, loop="asyncio", @@ -42,42 +20,28 @@ def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): ssl_keyfile=tls_ca_certificate_private_key_path, ssl_certfile=tls_ca_certificate_pem_path, ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - with no_ssl_verification(): - response = requests.get("https://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient(verify=False) as client: + response = await client.get("https://127.0.0.1:8000") assert response.status_code == 204 - thread.join() -@pytest.mark.skipif( - sys.platform.startswith("win"), reason="Skipping SSL test on Windows" -) -def test_run_chain(tls_certificate_pem_path): +@pytest.mark.asyncio +async def test_run_chain(tls_certificate_pem_path): config = Config( app=app, loop="asyncio", limit_max_requests=1, ssl_certfile=tls_certificate_pem_path, ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - with no_ssl_verification(): - response = requests.get("https://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient(verify=False) as client: + response = await client.get("https://127.0.0.1:8000") assert response.status_code == 204 - thread.join() -@pytest.mark.skipif( - sys.platform.startswith("win"), reason="Skipping SSL test on Windows" -) -def test_run_password( +@pytest.mark.asyncio +async def test_run_password( tls_ca_certificate_pem_path, tls_ca_certificate_private_key_encrypted_path ): config = Config( @@ -88,12 +52,7 @@ def test_run_password( ssl_certfile=tls_ca_certificate_pem_path, ssl_keyfile_password="uvicorn password for the win", ) - server = Server(config=config) - thread = threading.Thread(target=server.run) - thread.start() - while not server.started: - time.sleep(0.01) - with no_ssl_verification(): - response = requests.get("https://127.0.0.1:8000") + async with run_server(config): + async with httpx.AsyncClient(verify=False) as client: + response = await client.get("https://127.0.0.1:8000") assert response.status_code == 204 - thread.join() diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000..32ade817a --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,20 @@ +import asyncio + +try: + from contextlib import asynccontextmanager +except ImportError: # pragma: no cover + from async_generator import asynccontextmanager + +from uvicorn import Config, Server + + +@asynccontextmanager +async def run_server(config: Config, sockets=None): + server = Server(config=config) + cancel_handle = asyncio.ensure_future(server.serve(sockets=sockets)) + await asyncio.sleep(0.1) + try: + yield server + finally: + await server.shutdown() + cancel_handle.cancel() From 091571b2285c6a7e122012825d9f4be8de4e5a8e Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 29 Dec 2020 08:39:29 +0100 Subject: [PATCH 104/128] Fixes websockets implementation where handshake fails left tasks pending (#921) * Set handshake_started_event * Added comment --- uvicorn/protocols/websockets/websockets_impl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index 30be0dc2f..aa81915bd 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -135,6 +135,9 @@ def send_500_response(self): msg, ] self.transport.write(b"".join(content)) + # Allow handler task to terminate cleanly, as websockets doesn't cancel it by + # itself (see https://github.com/encode/uvicorn/issues/920) + self.handshake_started_event.set() async def ws_handler(self, protocol, path): """ From 36cfe87a4b4194a47e238123da4555230f750956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= Date: Tue, 29 Dec 2020 09:33:17 +0100 Subject: [PATCH 105/128] =?UTF-8?q?=F0=9F=90=9B=20Fix=20resetting=20signal?= =?UTF-8?q?s=20in=20Gunicorn=20UvicornWorker=20(#895)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Fix resetting signals in Gunicorn UvicornWorker to fix subprocesses that capture output having an incorrect returncode * Update uvicorn/workers.py Co-authored-by: Florimond Manca --- uvicorn/workers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/uvicorn/workers.py b/uvicorn/workers.py index 1ac036372..7c160dff6 100644 --- a/uvicorn/workers.py +++ b/uvicorn/workers.py @@ -1,5 +1,6 @@ import asyncio import logging +import signal from gunicorn.workers.base import Worker @@ -62,7 +63,11 @@ def init_process(self): super(UvicornWorker, self).init_process() def init_signals(self): - pass + # Reset signals so Gunicorn doesn't swallow subprocess return codes + # other signals are set up by Server.install_signal_handlers() + # See: https://github.com/encode/uvicorn/issues/894 + for s in self.SIGNALS: + signal.signal(s, signal.SIG_DFL) def run(self): self.config.app = self.wsgi From c4c76008bb22a67d0a4dd0d48270618e3bfc1aa0 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Tue, 29 Dec 2020 13:29:38 +0100 Subject: [PATCH 106/128] Add docs script (#923) --- scripts/docs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 scripts/docs diff --git a/scripts/docs b/scripts/docs new file mode 100755 index 000000000..3ece31f88 --- /dev/null +++ b/scripts/docs @@ -0,0 +1,10 @@ +#!/bin/sh -e + +PREFIX="" +if [ -d "venv" ] ; then + PREFIX="venv/bin/" +fi + +set -x + +${PREFIX}mkdocs "$@" From fa914bc8b3af38017f2438770616cab46f600141 Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Tue, 29 Dec 2020 17:40:53 +0100 Subject: [PATCH 107/128] Version 0.13.3 (#922) --- CHANGELOG.md | 8 ++++++++ uvicorn/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db0157c84..6b5f5cac1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log +## 0.13.3 - 2020-12-29 + +### Fixed + +- Prevent swallowing of return codes from `subprocess` when running with Gunicorn by properly resetting signals. (Pull #895) +- Tweak detection of app factories to be more robust. A warning is now logged when passing a factory without the `--factory` flag. (Pull #914) +- Properly clean tasks when handshake is aborted when running with `--ws websockets`. (Pull #921) + ## 0.13.2 - 2020-12-12 ### Fixed diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 6a72df40d..9392a5009 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.13.2" +__version__ = "0.13.3" __all__ = ["main", "run", "Config", "Server"] From 3da100fafa81581b6bdb56e05e9546a218ca984d Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 29 Dec 2020 20:42:01 +0100 Subject: [PATCH 108/128] Use ssl verification in tests (#928) * Use ssl verification ! * Better naming is always welcomed --- tests/conftest.py | 9 +++++++++ tests/test_ssl.py | 16 ++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 00428b982..a9c6dad0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +import ssl + import pytest import trustme from cryptography.hazmat.backends import default_backend @@ -50,3 +52,10 @@ def tls_ca_certificate_private_key_encrypted_path(tls_certificate_authority): def tls_certificate_pem_path(tls_certificate): with tls_certificate.private_key_and_cert_chain_pem.tempfile() as cert_pem: yield cert_pem + + +@pytest.fixture +def tls_ca_ssl_context(tls_certificate): + ssl_ctx = ssl.SSLContext() + tls_certificate.configure_cert(ssl_ctx) + return ssl_ctx diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 4cea0963b..a18069374 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -12,7 +12,9 @@ async def app(scope, receive, send): @pytest.mark.asyncio -async def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path): +async def test_run( + tls_ca_ssl_context, tls_ca_certificate_pem_path, tls_ca_certificate_private_key_path +): config = Config( app=app, loop="asyncio", @@ -21,13 +23,13 @@ async def test_run(tls_ca_certificate_pem_path, tls_ca_certificate_private_key_p ssl_certfile=tls_ca_certificate_pem_path, ) async with run_server(config): - async with httpx.AsyncClient(verify=False) as client: + async with httpx.AsyncClient(verify=tls_ca_ssl_context) as client: response = await client.get("https://127.0.0.1:8000") assert response.status_code == 204 @pytest.mark.asyncio -async def test_run_chain(tls_certificate_pem_path): +async def test_run_chain(tls_ca_ssl_context, tls_certificate_pem_path): config = Config( app=app, loop="asyncio", @@ -35,14 +37,16 @@ async def test_run_chain(tls_certificate_pem_path): ssl_certfile=tls_certificate_pem_path, ) async with run_server(config): - async with httpx.AsyncClient(verify=False) as client: + async with httpx.AsyncClient(verify=tls_ca_ssl_context) as client: response = await client.get("https://127.0.0.1:8000") assert response.status_code == 204 @pytest.mark.asyncio async def test_run_password( - tls_ca_certificate_pem_path, tls_ca_certificate_private_key_encrypted_path + tls_ca_ssl_context, + tls_ca_certificate_pem_path, + tls_ca_certificate_private_key_encrypted_path, ): config = Config( app=app, @@ -53,6 +57,6 @@ async def test_run_password( ssl_keyfile_password="uvicorn password for the win", ) async with run_server(config): - async with httpx.AsyncClient(verify=False) as client: + async with httpx.AsyncClient(verify=tls_ca_ssl_context) as client: response = await client.get("https://127.0.0.1:8000") assert response.status_code == 204 From a67ac860a0329138c29f6e208c85e0e2286d554e Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 29 Dec 2020 22:53:00 +0100 Subject: [PATCH 109/128] Remove requests dependency (#918) * Used async context manager instead of thread * Refactored test_ssl * Refactored test_main , last test become irrelevant * Same with test_websockets, now exceptions are bubbled ! * Update requirements.txt * Refactor test_trace_logging.py as well * Removed what appears to be dead code since we dont use threads * asynccontextmanager appears in 3.7, just removing it to check if CI passes * trace logging reworked * Attempt at 3.6 * Should be better with correct lib name * Removed requests dependency and pass it to httpx ! * Using same encoding as server * Blacked * Merge leftovers * No need for a client here * Another merge leftover, should have rebased * Removed test_client.py since we dont have it anymore ! * Direct use of httpx Headers * Verbose pytest * Throwing some debug stuff * Commenting the previous test just to see if the server close that should happen here is what causes the subsequent test to fail * Trying to cancel that task on connection_lost, is that legit ? * Ok so it shows this is indeed the task that is responsible for the flakiness but this feels a hack to cancel it here * Test script back * Candel handler task if connection lost and handshake not started ? * This look like the fix * Found the 2nd one, OMFG * Not part of that PR anymore --- requirements.txt | 1 - tests/client.py | 139 ------------------------ tests/middleware/test_debug.py | 51 ++++++--- tests/middleware/test_message_logger.py | 18 +-- tests/middleware/test_proxy_headers.py | 40 ++++--- tests/middleware/test_wsgi.py | 44 +++++--- tests/test_client.py | 28 ----- 7 files changed, 96 insertions(+), 225 deletions(-) delete mode 100644 tests/client.py delete mode 100644 tests/test_client.py diff --git a/requirements.txt b/requirements.txt index ae07f9654..a739e056d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ flake8 isort pytest pytest-mock -requests mypy trustme cryptography diff --git a/tests/client.py b/tests/client.py deleted file mode 100644 index 829d81b1e..000000000 --- a/tests/client.py +++ /dev/null @@ -1,139 +0,0 @@ -import asyncio -import io -import typing -from urllib.parse import unquote, urljoin, urlparse - -import requests - - -class _HeaderDict(requests.packages.urllib3._collections.HTTPHeaderDict): - def get_all(self, key, default): - return self.getheaders(key) - - -class _MockOriginalResponse(object): - """ - We have to jump through some hoops to present the response as if - it was made using urllib3. - """ - - def __init__(self, headers): - self.msg = _HeaderDict(headers) - self.closed = False - - def isclosed(self): - return self.closed - - -class _ASGIAdapter(requests.adapters.HTTPAdapter): - def __init__(self, app: typing.Callable, raise_server_exceptions=True) -> None: - self.app = app - self.raise_server_exceptions = raise_server_exceptions - - def send(self, request, *args, **kwargs): - scheme, netloc, path, params, query, fragement = urlparse(request.url) - if ":" in netloc: - host, port = netloc.split(":", 1) - port = int(port) - else: - host = netloc - port = {"http": 80, "ws": 80, "https": 443, "wss": 443}[scheme] - - # Include the 'host' header. - if "host" in request.headers: - headers = [] - elif port == 80: - headers = [[b"host", host.encode()]] - else: - headers = [[b"host", ("%s:%d" % (host, port)).encode()]] - - # Include other request headers. - headers += [ - [key.lower().encode(), value.encode()] - for key, value in request.headers.items() - ] - - scope = { - "type": "http", - "http_version": "1.1", - "method": request.method, - "path": unquote(path), - "root_path": "", - "scheme": scheme, - "query_string": query.encode(), - "headers": headers, - "client": ["testclient", 50000], - "server": [host, port], - } - - async def receive(): - body = request.body - if body is None: - body_bytes = b"" - else: - assert isinstance(body, bytes) - body_bytes = body - return {"type": "http.request", "body": body_bytes} - - async def send(message): - nonlocal raw_kwargs, response_started - - if message["type"] == "http.response.start": - raw_kwargs["version"] = 11 - raw_kwargs["status"] = message["status"] - raw_kwargs["headers"] = [ - (key.decode(), value.decode()) for key, value in message["headers"] - ] - raw_kwargs["preload_content"] = False - raw_kwargs["original_response"] = _MockOriginalResponse( - raw_kwargs["headers"] - ) - response_started = True - elif message["type"] == "http.response.body": - body = message.get("body", b"") - more_body = message.get("more_body", False) - raw_kwargs["body"].write(body) - if not more_body: - raw_kwargs["body"].seek(0) - - response_started = False - raw_kwargs = {"body": io.BytesIO()} - - loop = asyncio.get_event_loop() - - try: - loop.run_until_complete(self.app(scope, receive, send)) - except BaseException as exc: - if self.raise_server_exceptions: - raise exc from None - - raw = requests.packages.urllib3.HTTPResponse(**raw_kwargs) - return self.build_response(request, raw) - - -class _TestClient(requests.Session): - def __init__( - self, app: typing.Callable, base_url: str, raise_server_exceptions=True - ) -> None: - super(_TestClient, self).__init__() - adapter = _ASGIAdapter(app, raise_server_exceptions=raise_server_exceptions) - self.mount("http://", adapter) - self.mount("https://", adapter) - self.headers.update({"user-agent": "testclient"}) - self.base_url = base_url - - def request(self, method: str, url: str, **kwargs) -> requests.Response: - url = urljoin(self.base_url, url) - return super().request(method, url, **kwargs) - - -def TestClient( - app: typing.Callable, - base_url: str = "http://testserver", - raise_server_exceptions=True, -) -> _TestClient: - """ - We have to work around py.test discovery attempting to pick up - the `TestClient` class, by declaring this as a function. - """ - return _TestClient(app, base_url, raise_server_exceptions=raise_server_exceptions) diff --git a/tests/middleware/test_debug.py b/tests/middleware/test_debug.py index 60183dfaa..34b384208 100644 --- a/tests/middleware/test_debug.py +++ b/tests/middleware/test_debug.py @@ -1,54 +1,73 @@ -import asyncio - +import httpx import pytest -from tests.client import TestClient from uvicorn.middleware.debug import DebugMiddleware -def test_debug_text(): +@pytest.mark.asyncio +async def test_debug_text(): async def app(scope, receive, send): raise RuntimeError("Something went wrong") app = DebugMiddleware(app) - client = TestClient(app, raise_server_exceptions=False) - response = client.get("/") + transport = httpx.ASGITransport( + app=app, + raise_app_exceptions=False, + ) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + response = await client.get("/") + assert response.status_code == 500 assert response.headers["content-type"].startswith("text/plain") assert "RuntimeError" in response.text -def test_debug_html(): +@pytest.mark.asyncio +async def test_debug_html(): async def app(scope, receive, send): raise RuntimeError("Something went wrong") app = DebugMiddleware(app) - client = TestClient(app, raise_server_exceptions=False) - response = client.get("/", headers={"Accept": "text/html, */*"}) + transport = httpx.ASGITransport( + app=app, + raise_app_exceptions=False, + ) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + response = await client.get("/", headers={"Accept": "text/html, */*"}) assert response.status_code == 500 assert response.headers["content-type"].startswith("text/html") assert "RuntimeError" in response.text -def test_debug_after_response_sent(): +@pytest.mark.asyncio +async def test_debug_after_response_sent(): async def app(scope, receive, send): await send({"type": "http.response.start", "status": 204, "headers": []}) await send({"type": "http.response.body", "body": b"", "more_body": False}) raise RuntimeError("Something went wrong") app = DebugMiddleware(app) - client = TestClient(app, raise_server_exceptions=False) - response = client.get("/") + transport = httpx.ASGITransport( + app=app, + raise_app_exceptions=False, + ) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + response = await client.get("/") assert response.status_code == 204 assert response.content == b"" -def test_debug_not_http(): +@pytest.mark.asyncio +async def test_debug_not_http(): async def app(scope, send, receive): raise RuntimeError("Something went wrong") app = DebugMiddleware(app) - with pytest.raises(RuntimeError): - loop = asyncio.get_event_loop() - loop.run_until_complete(app({"type": "websocket"}, None, None)) + await app({"type": "websocket"}, None, None) diff --git a/tests/middleware/test_message_logger.py b/tests/middleware/test_message_logger.py index c18a30d20..e5d7de6a0 100644 --- a/tests/middleware/test_message_logger.py +++ b/tests/middleware/test_message_logger.py @@ -1,12 +1,13 @@ +import httpx import pytest -from tests.client import TestClient from uvicorn.middleware.message_logger import MessageLoggerMiddleware TRACE_LOG_LEVEL = 5 -def test_message_logger(caplog): +@pytest.mark.asyncio +async def test_message_logger(caplog): async def app(scope, receive, send): await receive() await send({"type": "http.response.start", "status": 200, "headers": []}) @@ -16,8 +17,8 @@ async def app(scope, receive, send): caplog.set_level(TRACE_LOG_LEVEL) app = MessageLoggerMiddleware(app) - client = TestClient(app) - response = client.get("/") + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + response = await client.get("/") assert response.status_code == 200 messages = [record.msg % record.args for record in caplog.records] assert sum(["ASGI [1] Started" in message for message in messages]) == 1 @@ -27,16 +28,17 @@ async def app(scope, receive, send): assert sum(["ASGI [1] Raised exception" in message for message in messages]) == 0 -def test_message_logger_exc(caplog): +@pytest.mark.asyncio +async def test_message_logger_exc(caplog): async def app(scope, receive, send): raise RuntimeError() caplog.set_level(TRACE_LOG_LEVEL, logger="uvicorn.asgi") caplog.set_level(TRACE_LOG_LEVEL) app = MessageLoggerMiddleware(app) - client = TestClient(app) - with pytest.raises(RuntimeError): - client.get("/") + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + with pytest.raises(RuntimeError): + await client.get("/") messages = [record.msg % record.args for record in caplog.records] assert sum(["ASGI [1] Started" in message for message in messages]) == 1 assert sum(["ASGI [1] Send" in message for message in messages]) == 0 diff --git a/tests/middleware/test_proxy_headers.py b/tests/middleware/test_proxy_headers.py index eef8dec5a..d20adcd88 100644 --- a/tests/middleware/test_proxy_headers.py +++ b/tests/middleware/test_proxy_headers.py @@ -1,4 +1,6 @@ -from tests.client import TestClient +import httpx +import pytest + from tests.response import Response from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware @@ -14,28 +16,34 @@ async def app(scope, receive, send): app = ProxyHeadersMiddleware(app, trusted_hosts="*") -def test_proxy_headers(): - client = TestClient(app) - headers = {"X-Forwarded-Proto": "https", "X-Forwarded-For": "1.2.3.4"} - response = client.get("/", headers=headers) +@pytest.mark.asyncio +async def test_proxy_headers(): + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + headers = {"X-Forwarded-Proto": "https", "X-Forwarded-For": "1.2.3.4"} + response = await client.get("/", headers=headers) assert response.status_code == 200 assert response.text == "Remote: https://1.2.3.4:0" -def test_proxy_headers_no_port(): - client = TestClient(app) - headers = {"X-Forwarded-Proto": "https", "X-Forwarded-For": "1.2.3.4"} - response = client.get("/", headers=headers) +@pytest.mark.asyncio +async def test_proxy_headers_no_port(): + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + headers = {"X-Forwarded-Proto": "https", "X-Forwarded-For": "1.2.3.4"} + response = await client.get("/", headers=headers) assert response.status_code == 200 assert response.text == "Remote: https://1.2.3.4:0" -def test_proxy_headers_invalid_x_forwarded_for(): - client = TestClient(app) - headers = { - "X-Forwarded-Proto": "https", - "X-Forwarded-For": "\xf0\xfd\xfd\xfd, 1.2.3.4", - } - response = client.get("/", headers=headers) +@pytest.mark.asyncio +async def test_proxy_headers_invalid_x_forwarded_for(): + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + headers = httpx.Headers( + { + "X-Forwarded-Proto": "https", + "X-Forwarded-For": "\xf0\xfd\xfd\xfd, 1.2.3.4", + }, + encoding="latin-1", + ) + response = await client.get("/", headers=headers) assert response.status_code == 200 assert response.text == "Remote: https://1.2.3.4:0" diff --git a/tests/middleware/test_wsgi.py b/tests/middleware/test_wsgi.py index a3acdcd32..dd4c31161 100644 --- a/tests/middleware/test_wsgi.py +++ b/tests/middleware/test_wsgi.py @@ -1,8 +1,8 @@ import sys +import httpx import pytest -from tests.client import TestClient from uvicorn.middleware.wsgi import WSGIMiddleware @@ -46,41 +46,51 @@ def return_exc_info(environ, start_response): return [output] -def test_wsgi_get(): +@pytest.mark.asyncio +async def test_wsgi_get(): app = WSGIMiddleware(hello_world) - client = TestClient(app) - response = client.get("/") + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + response = await client.get("/") assert response.status_code == 200 assert response.text == "Hello World!\n" -def test_wsgi_post(): +@pytest.mark.asyncio +async def test_wsgi_post(): app = WSGIMiddleware(echo_body) - client = TestClient(app) - response = client.post("/", json={"example": 123}) + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + response = await client.post("/", json={"example": 123}) assert response.status_code == 200 assert response.text == '{"example": 123}' -def test_wsgi_exception(): +@pytest.mark.asyncio +async def test_wsgi_exception(): # Note that we're testing the WSGI app directly here. # The HTTP protocol implementations would catch this error and return 500. app = WSGIMiddleware(raise_exception) - client = TestClient(app) - with pytest.raises(RuntimeError): - client.get("/") + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + with pytest.raises(RuntimeError): + await client.get("/") -def test_wsgi_exc_info(): +@pytest.mark.asyncio +async def test_wsgi_exc_info(): # Note that we're testing the WSGI app directly here. # The HTTP protocol implementations would catch this error and return 500. app = WSGIMiddleware(return_exc_info) - client = TestClient(app) - with pytest.raises(RuntimeError): - response = client.get("/") + async with httpx.AsyncClient(app=app, base_url="http://testserver") as client: + with pytest.raises(RuntimeError): + response = await client.get("/") app = WSGIMiddleware(return_exc_info) - client = TestClient(app, raise_server_exceptions=False) - response = client.get("/") + transport = httpx.ASGITransport( + app=app, + raise_app_exceptions=False, + ) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + response = await client.get("/") assert response.status_code == 500 assert response.text == "Internal Server Error" diff --git a/tests/test_client.py b/tests/test_client.py deleted file mode 100644 index 98edfdba0..000000000 --- a/tests/test_client.py +++ /dev/null @@ -1,28 +0,0 @@ -from tests.client import TestClient - - -async def hello_world(scope, receive, send): - await send( - { - "type": "http.response.start", - "status": 200, - "headers": [(b"content-type", b"text/plain")], - } - ) - await send( - {"type": "http.response.body", "body": b"hello, world", "more_body": False} - ) - - -def test_explicit_base_url(): - client = TestClient(hello_world, base_url="http://testserver:321") - response = client.get("/") - assert response.status_code == 200 - assert response.text == "hello, world" - - -def test_explicit_host(): - client = TestClient(hello_world) - response = client.get("/", headers={"host": "example.org"}) - assert response.status_code == 200 - assert response.text == "hello, world" From db4683fffd8650eea81e10a1987a2115b9770695 Mon Sep 17 00:00:00 2001 From: Kevin Michel <18656438+kmichel-sereema@users.noreply.github.com> Date: Fri, 1 Jan 2021 16:14:49 +0100 Subject: [PATCH 110/128] Enabled permessage-deflate extension in websockets (#764) The permessage deflate extension for the websocket protocol was already enabled when using the wsproto implementation. This enables it for the websockets implementation and add tests for both implementations --- tests/protocols/test_websocket.py | 20 +++++++++++++++++++ .../protocols/websockets/websockets_impl.py | 7 ++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 24bc285ff..57a8c7af6 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -7,11 +7,13 @@ try: import websockets + from websockets.extensions.permessage_deflate import ClientPerMessageDeflateFactory from uvicorn.protocols.websockets.websockets_impl import WebSocketProtocol except ImportError: # pragma: nocover websockets = None WebSocketProtocol = None + ClientPerMessageDeflateFactory = None WS_PROTOCOLS = [p for p in [WSProtocol, WebSocketProtocol] if p is not None] @@ -87,6 +89,24 @@ async def open_connection(url): assert is_open +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) +async def test_supports_permessage_deflate_extension(protocol_cls): + class App(WebSocketResponse): + async def websocket_connect(self, message): + await self.send({"type": "websocket.accept"}) + + async def open_connection(url): + extension_factories = [ClientPerMessageDeflateFactory()] + async with websockets.connect(url, extensions=extension_factories) as websocket: + return [extension.name for extension in websocket.extensions] + + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + extension_names = await open_connection("ws://127.0.0.1:8000") + assert "permessage-deflate" in extension_names + + @pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) async def test_close_connection(protocol_cls): diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index aa81915bd..6057fec4a 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -4,6 +4,7 @@ from urllib.parse import unquote import websockets +from websockets.extensions.permessage_deflate import ServerPerMessageDeflateFactory from uvicorn.protocols.utils import get_local_addr, get_remote_addr, is_ssl @@ -54,7 +55,11 @@ def __init__(self, config, server_state, _loop=None): self.ws_server = Server() - super().__init__(ws_handler=self.ws_handler, ws_server=self.ws_server) + super().__init__( + ws_handler=self.ws_handler, + ws_server=self.ws_server, + extensions=[ServerPerMessageDeflateFactory()], + ) def connection_made(self, transport): self.connections.add(self) From 320fd6d975d47257aecde808d602c9a57fb74cba Mon Sep 17 00:00:00 2001 From: Florimond Manca Date: Sun, 3 Jan 2021 11:55:24 +0100 Subject: [PATCH 111/128] Isolate server started message (#930) --- uvicorn/server.py | 58 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/uvicorn/server.py b/uvicorn/server.py index d289ee599..b54be77e8 100644 --- a/uvicorn/server.py +++ b/uvicorn/server.py @@ -9,6 +9,7 @@ import threading import time from email.utils import formatdate +from typing import List import click @@ -110,6 +111,7 @@ def _share_socket(sock: socket) -> socket: create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog ) self.servers.append(server) + listeners = sockets elif config.fd is not None: # Use an existing socket, from a file descriptor. @@ -117,8 +119,8 @@ def _share_socket(sock: socket) -> socket: server = await loop.create_server( create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog ) - message = "Uvicorn running on socket %s (Press CTRL+C to quit)" - logger.info(message % str(sock.getsockname())) + assert server.sockets is not None # mypy + listeners = server.sockets self.servers = [server] elif config.uds is not None: @@ -130,17 +132,12 @@ def _share_socket(sock: socket) -> socket: create_protocol, path=config.uds, ssl=config.ssl, backlog=config.backlog ) os.chmod(config.uds, uds_perms) - message = "Uvicorn running on unix socket %s (Press CTRL+C to quit)" - logger.info(message % config.uds) + assert server.sockets is not None # mypy + listeners = server.sockets self.servers = [server] else: # Standard case. Create a socket from a host/port pair. - addr_format = "%s://%s:%d" - if config.host and ":" in config.host: - # It's an IPv6 address. - addr_format = "%s://[%s]:%d" - try: server = await loop.create_server( create_protocol, @@ -153,9 +150,45 @@ def _share_socket(sock: socket) -> socket: logger.error(exc) await self.lifespan.shutdown() sys.exit(1) + assert server.sockets is not None # mypy + listeners = server.sockets + self.servers = [server] + + if sockets is None: + self._log_started_message(listeners) + else: + # We're most likely running multiple workers, so a message has already been + # logged by `config.bind_socket()`. + pass + + self.started = True + + def _log_started_message(self, listeners: List[socket.SocketType]) -> None: + config = self.config + + if config.fd is not None: + sock = listeners[0] + logger.info( + "Uvicorn running on socket %s (Press CTRL+C to quit)", + sock.getsockname(), + ) + + elif config.uds is not None: + logger.info( + "Uvicorn running on unix socket %s (Press CTRL+C to quit)", config.uds + ) + + else: + addr_format = "%s://%s:%d" + host = "0.0.0.0" if config.host is None else config.host + if ":" in host: + # It's an IPv6 address. + addr_format = "%s://[%s]:%d" + port = config.port if port == 0: - port = server.sockets[0].getsockname()[1] + port = listeners[0].getpeername()[1] + protocol_name = "https" if config.ssl else "http" message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" color_message = ( @@ -166,13 +199,10 @@ def _share_socket(sock: socket) -> socket: logger.info( message, protocol_name, - config.host, + host, port, extra={"color_message": color_message}, ) - self.servers = [server] - - self.started = True async def main_loop(self): counter = 0 From bd1a0962d36c66e823e13012df14a0fe7e36cc9c Mon Sep 17 00:00:00 2001 From: Roald Storm Date: Mon, 25 Jan 2021 09:39:48 +0100 Subject: [PATCH 112/128] Return 'connection: close' header in response (#721) According to https://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html a server should return the 'connection: close' header in the response when a request has that header. Resolves #720 --- tests/test_main.py | 15 +++++++++++++++ uvicorn/protocols/http/h11_impl.py | 5 +++++ uvicorn/protocols/http/httptools_impl.py | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 5a5f3fd88..a4c5349ec 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -11,6 +11,21 @@ async def app(scope, receive, send): await send({"type": "http.response.body", "body": b"", "more_body": False}) +@pytest.mark.asyncio +async def test_return_close_header(): + config = Config(app=app, host="localhost", loop="asyncio", limit_max_requests=1) + async with run_server(config): + async with httpx.AsyncClient() as client: + response = await client.get( + "http://127.0.0.1:8000", headers={"connection": "close"} + ) + + assert response.status_code == 204 + assert ( + "connection" in response.headers and response.headers["connection"] == "close" + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "host, url", diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index 5d2bad4a1..90850dabb 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -25,6 +25,8 @@ def _get_status_phrase(status_code): status_code: _get_status_phrase(status_code) for status_code in range(100, 600) } +CLOSE_HEADER = (b"connection", b"close") + HIGH_WATER_LIMIT = 65536 TRACE_LOG_LEVEL = 5 @@ -452,6 +454,9 @@ async def send(self, message): status_code = message["status"] headers = self.default_headers + message.get("headers", []) + if CLOSE_HEADER in self.scope["headers"] and CLOSE_HEADER not in headers: + headers = headers + [CLOSE_HEADER] + if self.access_log: self.access_logger.info( '%s - "%s %s HTTP/%s" %d', diff --git a/uvicorn/protocols/http/httptools_impl.py b/uvicorn/protocols/http/httptools_impl.py index 7be086562..6dcb6e02a 100644 --- a/uvicorn/protocols/http/httptools_impl.py +++ b/uvicorn/protocols/http/httptools_impl.py @@ -30,6 +30,8 @@ def _get_status_line(status_code): status_code: _get_status_line(status_code) for status_code in range(100, 600) } +CLOSE_HEADER = (b"connection", b"close") + HIGH_WATER_LIMIT = 65536 TRACE_LOG_LEVEL = 5 @@ -454,6 +456,9 @@ async def send(self, message): status_code = message["status"] headers = self.default_headers + list(message.get("headers", [])) + if CLOSE_HEADER in self.scope["headers"] and CLOSE_HEADER not in headers: + headers = headers + [CLOSE_HEADER] + if self.access_log: self.access_logger.info( '%s - "%s %s HTTP/%s" %d', From 61a6cabb4580e1c923df396eac264803f599412c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Mazzucotelli?= Date: Mon, 25 Jan 2021 12:35:40 +0100 Subject: [PATCH 113/128] Document the default value of 1 for workers (#940) (#943) --- docs/deployment.md | 2 +- docs/index.md | 2 +- docs/settings.md | 2 +- uvicorn/main.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index cec0d8ef8..8aabe1b18 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -45,7 +45,7 @@ Options: --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if - available. Not valid with --reload. + available, or 1. Not valid with --reload. --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: diff --git a/docs/index.md b/docs/index.md index be1a096ca..ae587f2ee 100644 --- a/docs/index.md +++ b/docs/index.md @@ -115,7 +115,7 @@ Options: --workers INTEGER Number of worker processes. Defaults to the $WEB_CONCURRENCY environment variable if - available. Not valid with --reload. + available, or 1. Not valid with --reload. --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: diff --git a/docs/settings.md b/docs/settings.md index 2a717ab93..8b8398f83 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -24,7 +24,7 @@ equivalent keyword arguments, eg. `uvicorn.run("example:app", port=5000, reload= ## Production -* `--workers ` - Use multiple worker processes. Defaults to the value of the `$WEB_CONCURRENCY` environment variable. +* `--workers ` - Use multiple worker processes. Defaults to the `$WEB_CONCURRENCY` environment variable if available, or 1. ## Logging diff --git a/uvicorn/main.py b/uvicorn/main.py index 6bfc631f0..d7e866ec4 100644 --- a/uvicorn/main.py +++ b/uvicorn/main.py @@ -90,7 +90,7 @@ def print_version(ctx, param, value): default=None, type=int, help="Number of worker processes. Defaults to the $WEB_CONCURRENCY environment" - " variable if available. Not valid with --reload.", + " variable if available, or 1. Not valid with --reload.", ) @click.option( "--loop", From cf0b051196ad949af351a4b4be8f9bafa2884759 Mon Sep 17 00:00:00 2001 From: euri10 Date: Sun, 31 Jan 2021 13:32:50 +0100 Subject: [PATCH 114/128] Relax watchgod up bound (#946) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 61ea105e8..5cd54d7c5 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ def get_packages(package): "httptools==0.1.* ;" + env_marker_cpython, "uvloop>=0.14.0 ;" + env_marker_cpython, "colorama>=0.4;" + env_marker_win, - "watchgod>=0.6,<0.7", + "watchgod>=0.6", "python-dotenv>=0.13", "PyYAML>=5.1", ] From 1291b5da961d30725665d69cac86434412905d51 Mon Sep 17 00:00:00 2001 From: Jose Eduardo Date: Wed, 10 Feb 2021 13:55:04 +0000 Subject: [PATCH 115/128] Docs: Nginx + websockets (#948) Nginx requires that we set the Connection/Upgrade headers explicitly in the config, otherwise Nginx assumes that they shouldn't be forwarded. More info: https://nginx.org/en/docs/http/websocket.html --- docs/deployment.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/deployment.md b/docs/deployment.md index 8aabe1b18..3de15e6d1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -247,6 +247,8 @@ You should ensure that the `X-Forwarded-For` and `X-Forwarded-Proto` headers are Here's how a simple Nginx configuration might look. This example includes setting proxy headers, and using a UNIX domain socket to communicate with the application server. +It also includes some basic configuration to forward websocket connections. For more info on this, check [Nginx recommendations][nginx_websocket]. + ```conf http { server { @@ -259,6 +261,8 @@ http { proxy_set_header Host $http_host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; proxy_redirect off; proxy_buffering off; proxy_pass http://uvicorn; @@ -269,6 +273,11 @@ http { root /path/to/app/static; } } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } upstream uvicorn { server unix:/tmp/uvicorn.sock; @@ -309,5 +318,6 @@ It also possible to use certificates with uvicorn's worker for gunicorn $ gunicorn --keyfile=./key.pem --certfile=./cert.pem -k uvicorn.workers.UvicornWorker example:app ``` +[nginx_websocket]: https://nginx.org/en/docs/http/websocket.html [letsencrypt]: https://letsencrypt.org/ [mkcert]: https://github.com/FiloSottile/mkcert From bf7dac3112bc6ce59561703c80a619a287d50fa6 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Wed, 10 Feb 2021 11:32:15 -0300 Subject: [PATCH 116/128] fix: pin uvloop to the lastest Python 3.6 supported version (#952) * fix: pin uvloop to the lastest Python 3.6 supported version * fix multiple python versions --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5cd54d7c5..5f4172216 100755 --- a/setup.py +++ b/setup.py @@ -49,10 +49,12 @@ def get_packages(package): "typing-extensions;" + env_marker_below_38, ] + extra_requirements = [ "websockets==8.*", "httptools==0.1.* ;" + env_marker_cpython, - "uvloop>=0.14.0 ;" + env_marker_cpython, + "uvloop==0.14.0; python_version == '3.6' and " + env_marker_cpython, + "uvloop>=0.14.0; python_version >= '3.7' and " + env_marker_cpython, "colorama>=0.4;" + env_marker_win, "watchgod>=0.6", "python-dotenv>=0.13", From 9d51e1cfa68ee6e4ee3fb8a23b8599bb44c3985a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 20 Feb 2021 07:11:39 +0000 Subject: [PATCH 117/128] unpin uvloop (#959) Now that uvloop sets python_requires the pinning isn't needed, but we do need to ignore the packages with missing python_requires https://github.com/MagicStack/uvloop/pull/401 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5f4172216..73f1771d1 100755 --- a/setup.py +++ b/setup.py @@ -53,8 +53,7 @@ def get_packages(package): extra_requirements = [ "websockets==8.*", "httptools==0.1.* ;" + env_marker_cpython, - "uvloop==0.14.0; python_version == '3.6' and " + env_marker_cpython, - "uvloop>=0.14.0; python_version >= '3.7' and " + env_marker_cpython, + "uvloop>=0.14.0,!=0.15.0,!=0.15.1; " + env_marker_cpython, "colorama>=0.4;" + env_marker_win, "watchgod>=0.6", "python-dotenv>=0.13", From 2b1a67f97e7adcfe70b587a20a8f1516451ba542 Mon Sep 17 00:00:00 2001 From: CoolSpring8 Date: Sun, 21 Feb 2021 00:41:00 +0800 Subject: [PATCH 118/128] Fix wsgi middleware PATH_INFO encoding (#962) * Fix wsgi middleware PATH_INFO encoding * Add tests for wsgi PATH_INFO encoding --- tests/middleware/test_wsgi.py | 16 +++++++++++++++- uvicorn/middleware/wsgi.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/middleware/test_wsgi.py b/tests/middleware/test_wsgi.py index dd4c31161..b2ea7972a 100644 --- a/tests/middleware/test_wsgi.py +++ b/tests/middleware/test_wsgi.py @@ -3,7 +3,7 @@ import httpx import pytest -from uvicorn.middleware.wsgi import WSGIMiddleware +from uvicorn.middleware.wsgi import WSGIMiddleware, build_environ def hello_world(environ, start_response): @@ -94,3 +94,17 @@ async def test_wsgi_exc_info(): response = await client.get("/") assert response.status_code == 500 assert response.text == "Internal Server Error" + + +def test_build_environ_encoding(): + scope = { + "type": "http", + "http_version": "1.1", + "method": "GET", + "path": "/文", + "root_path": "/文", + "query_string": b"a=123&b=456", + "headers": [], + } + environ = build_environ(scope, b"", b"") + assert environ["PATH_INFO"] == "/文".encode("utf8").decode("latin-1") diff --git a/uvicorn/middleware/wsgi.py b/uvicorn/middleware/wsgi.py index e5346ddcf..fca747ed7 100644 --- a/uvicorn/middleware/wsgi.py +++ b/uvicorn/middleware/wsgi.py @@ -11,7 +11,7 @@ def build_environ(scope, message, body): environ = { "REQUEST_METHOD": scope["method"], "SCRIPT_NAME": "", - "PATH_INFO": scope["path"], + "PATH_INFO": scope["path"].encode("utf8").decode("latin1"), "QUERY_STRING": scope["query_string"].decode("ascii"), "SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"], "wsgi.version": (1, 0), From 2a7634d190787aac8fce757146206565feba30fa Mon Sep 17 00:00:00 2001 From: euri10 Date: Sat, 20 Feb 2021 17:48:34 +0100 Subject: [PATCH 119/128] version 0.13.4 (#958) * v0.13.4 * v0.13.4 * Fixed commit * Adapted with latest uvloop fix commit * Adapted with latest uvloop fix commit * Sneam wsgi fix --- CHANGELOG.md | 15 +++++++++++++++ uvicorn/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b5f5cac1..232848452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Change Log +## 0.13.4 - 2021-02-20 + +### Fixed + +- Fixed wsgi middleware PATH_INFO encoding (#962) 2/20/21 +- Fixed uvloop dependency (#952) 2/10/21 then (#959) 2/20/21 +- Relax watchgod up bound (#946) 1/31/21 +- Return 'connection: close' header in response (#721) 1/25/21 + +### Added: + +- Docs: Nginx + websockets (#948) 2/10/21 +- Document the default value of 1 for workers (#940) (#943) 1/25/21 +- Enabled permessage-deflate extension in websockets (#764) 1/1/21 + ## 0.13.3 - 2020-12-29 ### Fixed diff --git a/uvicorn/__init__.py b/uvicorn/__init__.py index 9392a5009..63b79ff12 100644 --- a/uvicorn/__init__.py +++ b/uvicorn/__init__.py @@ -1,5 +1,5 @@ from uvicorn.config import Config from uvicorn.main import Server, main, run -__version__ = "0.13.3" +__version__ = "0.13.4" __all__ = ["main", "run", "Config", "Server"] From d716ad0930a5a3ec21b8fe6674fc3fa4458636c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Lowas-Rzechonek?= Date: Wed, 24 Feb 2021 08:47:34 +0100 Subject: [PATCH 120/128] Support 'reason' field in 'websocket.close' messages (#957) --- tests/protocols/test_websocket.py | 17 ++++++++++++++--- uvicorn/protocols/websockets/websockets_impl.py | 3 ++- uvicorn/protocols/websockets/wsproto_impl.py | 5 ++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 57a8c7af6..00e09696c 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -377,14 +377,24 @@ async def connect(url): @pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", WS_PROTOCOLS) -async def test_app_close(protocol_cls): +@pytest.mark.parametrize("code", [None, 1000, 1001]) +@pytest.mark.parametrize("reason", [None, "test"]) +async def test_app_close(protocol_cls, code, reason): async def app(scope, receive, send): while True: message = await receive() if message["type"] == "websocket.connect": await send({"type": "websocket.accept"}) elif message["type"] == "websocket.receive": - await send({"type": "websocket.close"}) + reply = {"type": "websocket.close"} + + if code is not None: + reply["code"] = code + + if reason is not None: + reply["reason"] = reason + + await send(reply) elif message["type"] == "websocket.disconnect": break @@ -398,7 +408,8 @@ async def websocket_session(url): async with run_server(config): with pytest.raises(websockets.exceptions.ConnectionClosed) as exc_info: await websocket_session("ws://127.0.0.1:8000") - assert exc_info.value.code == 1000 + assert exc_info.value.code == (code or 1000) + assert exc_info.value.reason == (reason or "") @pytest.mark.asyncio diff --git a/uvicorn/protocols/websockets/websockets_impl.py b/uvicorn/protocols/websockets/websockets_impl.py index 6057fec4a..4afd72f10 100644 --- a/uvicorn/protocols/websockets/websockets_impl.py +++ b/uvicorn/protocols/websockets/websockets_impl.py @@ -224,7 +224,8 @@ async def asgi_send(self, message): elif message_type == "websocket.close": code = message.get("code", 1000) - await self.close(code) + reason = message.get("reason", "") + await self.close(code, reason) self.closed_event.set() else: diff --git a/uvicorn/protocols/websockets/wsproto_impl.py b/uvicorn/protocols/websockets/wsproto_impl.py index 456af1370..6d15f8342 100644 --- a/uvicorn/protocols/websockets/wsproto_impl.py +++ b/uvicorn/protocols/websockets/wsproto_impl.py @@ -279,8 +279,11 @@ async def send(self, message): elif message_type == "websocket.close": self.close_sent = True code = message.get("code", 1000) + reason = message.get("reason", "") self.queue.put_nowait({"type": "websocket.disconnect", "code": code}) - output = self.conn.send(wsproto.events.CloseConnection(code=code)) + output = self.conn.send( + wsproto.events.CloseConnection(code=code, reason=reason) + ) if not self.transport.is_closing(): self.transport.write(output) self.transport.close() From 0a35a50318d373655033416e84916beacf8ddac7 Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 25 Feb 2021 11:55:37 +0100 Subject: [PATCH 121/128] Implemented lifespan.shutdown.failed (#755) * Added test and implemented lifespan.shutdown.failed * Move new test to bottom --- tests/test_lifespan.py | 30 ++++++++++++++++++++++++++++++ uvicorn/lifespan/on.py | 21 +++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py index de584e1a8..2c83199bd 100644 --- a/tests/test_lifespan.py +++ b/tests/test_lifespan.py @@ -197,3 +197,33 @@ async def test(): loop = asyncio.new_event_loop() loop.run_until_complete(test()) + + +@pytest.mark.parametrize("mode", ("auto", "on")) +@pytest.mark.parametrize("raise_exception", (True, False)) +def test_lifespan_with_failed_shutdown(mode, raise_exception): + async def app(scope, receive, send): + message = await receive() + assert message["type"] == "lifespan.startup" + await send({"type": "lifespan.startup.complete"}) + message = await receive() + assert message["type"] == "lifespan.shutdown" + await send({"type": "lifespan.shutdown.failed"}) + + if raise_exception: + # App should be able to re-raise an exception if startup failed. + raise RuntimeError() + + async def test(): + config = Config(app=app, lifespan=mode) + lifespan = LifespanOn(config) + + await lifespan.startup() + assert not lifespan.startup_failed + await lifespan.shutdown() + assert lifespan.shutdown_failed + assert lifespan.error_occured is raise_exception + assert lifespan.should_exit + + loop = asyncio.new_event_loop() + loop.run_until_complete(test()) diff --git a/uvicorn/lifespan/on.py b/uvicorn/lifespan/on.py index 0f0644fef..a8deb7644 100644 --- a/uvicorn/lifespan/on.py +++ b/uvicorn/lifespan/on.py @@ -20,6 +20,7 @@ def __init__(self, config: Config) -> None: self.receive_queue: "Queue[LifespanReceiveMessage]" = asyncio.Queue() self.error_occured = False self.startup_failed = False + self.shutdown_failed = False self.should_exit = False async def startup(self) -> None: @@ -43,7 +44,14 @@ async def shutdown(self) -> None: self.logger.info("Waiting for application shutdown.") await self.receive_queue.put({"type": "lifespan.shutdown"}) await self.shutdown_event.wait() - self.logger.info("Application shutdown complete.") + + if self.shutdown_failed or ( + self.error_occured and self.config.lifespan == "on" + ): + self.logger.error("Application shutdown failed. Exiting.") + self.should_exit = True + else: + self.logger.info("Application shutdown complete.") async def main(self) -> None: try: @@ -56,7 +64,7 @@ async def main(self) -> None: except BaseException as exc: self.asgi = None self.error_occured = True - if self.startup_failed: + if self.startup_failed or self.shutdown_failed: return if self.config.lifespan == "auto": msg = "ASGI 'lifespan' protocol appears unsupported." @@ -73,6 +81,7 @@ async def send(self, message: LifespanSendMessage) -> None: "lifespan.startup.complete", "lifespan.startup.failed", "lifespan.shutdown.complete", + "lifespan.shutdown.failed", ) if message["type"] == "lifespan.startup.complete": @@ -93,5 +102,13 @@ async def send(self, message: LifespanSendMessage) -> None: assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR self.shutdown_event.set() + elif message["type"] == "lifespan.shutdown.failed": + assert self.startup_event.is_set(), STATE_TRANSITION_ERROR + assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR + self.shutdown_event.set() + self.shutdown_failed = True + if message.get("message"): + self.logger.error(message["message"]) + async def receive(self) -> LifespanReceiveMessage: return await self.receive_queue.get() From 9dc5a43209fe081ba3e74135189252bfddf75587 Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 25 Feb 2021 11:56:00 +0100 Subject: [PATCH 122/128] Up wsproto to 1.0.0 (#892) * Up wsproto to 1.0.0 * We should test against both ws protocols * Invalid upgrade in case of wsproto need ws version at least * Removed paramtrized test for upgrade with websockets, but we should have one * Removed leftovers * Blacked * Lint * No need for that change * No need for that change 2 * No need for that change 3 * No need for that change 4 * All you need is love * Not changing that too * Not changing that too 2 * Same upgrade request changed --- requirements.txt | 2 +- tests/protocols/test_http.py | 1 + tests/protocols/test_websocket.py | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a739e056d..bcc6b7dcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -e .[standard] # Explicit optionals -wsproto==0.15.* +wsproto==1.0.* # Packaging twine diff --git a/tests/protocols/test_http.py b/tests/protocols/test_http.py index f35a965d6..25110fc66 100644 --- a/tests/protocols/test_http.py +++ b/tests/protocols/test_http.py @@ -67,6 +67,7 @@ b"Host: example.org", b"Connection: upgrade", b"Upgrade: websocket", + b"Sec-WebSocket-Version: 11", b"", b"", ] diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 00e09696c..5f6a617cd 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -53,7 +53,11 @@ def app(scope): async with httpx.AsyncClient() as client: response = await client.get( "http://127.0.0.1:8000", - headers={"upgrade": "websocket", "connection": "upgrade"}, + headers={ + "upgrade": "websocket", + "connection": "upgrade", + "sec-webSocket-version": "11", + }, timeout=5, ) if response.status_code == 426: From 942deb9f8022ea7fa83e4816d26f48dd03df103b Mon Sep 17 00:00:00 2001 From: euri10 Date: Thu, 25 Feb 2021 17:01:19 +0100 Subject: [PATCH 123/128] Improve user feedback if no ws library installed (#926) * Note about ws not by default * Add a warning message * Removed index prose * Add warning in h11 which is where you'll be with uvicorn installed without ws libraries * Better warning message Co-authored-by: Florimond Manca * Seperate upgrade warning from libraries warning Co-authored-by: Florimond Manca --- uvicorn/protocols/http/h11_impl.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/uvicorn/protocols/http/h11_impl.py b/uvicorn/protocols/http/h11_impl.py index 90850dabb..dc887cd6e 100644 --- a/uvicorn/protocols/http/h11_impl.py +++ b/uvicorn/protocols/http/h11_impl.py @@ -268,6 +268,13 @@ def handle_upgrade(self, event): if upgrade_value != b"websocket" or self.ws_protocol_class is None: msg = "Unsupported upgrade request." self.logger.warning(msg) + + from uvicorn.protocols.websockets.auto import AutoWebSocketsProtocol + + if AutoWebSocketsProtocol is None: + msg = "No supported WebSocket library detected. Please use 'pip install uvicorn[standard]', or install 'websockets' or 'wsproto' manually." # noqa: E501 + self.logger.warning(msg) + reason = STATUS_PHRASES[400] headers = [ (b"content-type", b"text/plain; charset=utf-8"), From 83303ffe492e95fd60b9ec07a5b9c8de88a86e34 Mon Sep 17 00:00:00 2001 From: "Matthew D. Scholefield" Date: Thu, 4 Mar 2021 06:18:18 -0600 Subject: [PATCH 124/128] Prevent garbage collection of main lifespan task (#972) --- uvicorn/lifespan/on.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uvicorn/lifespan/on.py b/uvicorn/lifespan/on.py index a8deb7644..ec68189cd 100644 --- a/uvicorn/lifespan/on.py +++ b/uvicorn/lifespan/on.py @@ -27,7 +27,9 @@ async def startup(self) -> None: self.logger.info("Waiting for application startup.") loop = asyncio.get_event_loop() - loop.create_task(self.main()) + main_lifespan_task = loop.create_task(self.main()) # noqa: F841 + # Keep a hard reference to prevent garbage collection + # See https://github.com/encode/uvicorn/pull/972 await self.receive_queue.put({"type": "lifespan.startup"}) await self.startup_event.wait() From d5d62d49cb0ab1a46a4704d5f489ad1964e18747 Mon Sep 17 00:00:00 2001 From: euri10 Date: Fri, 5 Mar 2021 13:18:43 +0100 Subject: [PATCH 125/128] Fix socket port 0 (#975) --- uvicorn/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvicorn/server.py b/uvicorn/server.py index b54be77e8..fa174b9de 100644 --- a/uvicorn/server.py +++ b/uvicorn/server.py @@ -187,7 +187,7 @@ def _log_started_message(self, listeners: List[socket.SocketType]) -> None: port = config.port if port == 0: - port = listeners[0].getpeername()[1] + port = listeners[0].getsockname()[1] protocol_name = "https" if config.ssl else "http" message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" From 1e3eb1e1c2046cecf74e5e16fc37c5b1007708ce Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Sun, 7 Mar 2021 23:39:32 -0800 Subject: [PATCH 126/128] tests: add log coverage on lifespan failed shutdown event (#981) --- tests/test_lifespan.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py index 2c83199bd..462d4f6ae 100644 --- a/tests/test_lifespan.py +++ b/tests/test_lifespan.py @@ -201,14 +201,16 @@ async def test(): @pytest.mark.parametrize("mode", ("auto", "on")) @pytest.mark.parametrize("raise_exception", (True, False)) -def test_lifespan_with_failed_shutdown(mode, raise_exception): +def test_lifespan_with_failed_shutdown(mode, raise_exception, caplog): async def app(scope, receive, send): message = await receive() assert message["type"] == "lifespan.startup" await send({"type": "lifespan.startup.complete"}) message = await receive() assert message["type"] == "lifespan.shutdown" - await send({"type": "lifespan.shutdown.failed"}) + await send( + {"type": "lifespan.shutdown.failed", "message": "the lifespan event failed"} + ) if raise_exception: # App should be able to re-raise an exception if startup failed. @@ -227,3 +229,10 @@ async def test(): loop = asyncio.new_event_loop() loop.run_until_complete(test()) + error_messages = [ + record.message + for record in caplog.records + if record.name == "uvicorn.error" and record.levelname == "ERROR" + ] + assert "the lifespan event failed" in error_messages.pop(0) + assert "Application shutdown failed. Exiting." in error_messages.pop(0) From 896788852ec104f51d014002549f4c2dca0e0705 Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Mon, 8 Mar 2021 02:34:51 -0800 Subject: [PATCH 127/128] tests: add ssl coverage (#982) * tests: add ssl coverage * add ssl_ca_certs to all ssl tests --- tests/conftest.py | 10 +++++----- tests/test_ssl.py | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a9c6dad0a..a5e5a8e12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ def tls_certificate_authority() -> trustme.CA: @pytest.fixture -def tls_certificate(tls_certificate_authority): +def tls_certificate(tls_certificate_authority: trustme.CA) -> trustme.LeafCert: return tls_certificate_authority.issue_server_cert( "localhost", "127.0.0.1", @@ -21,13 +21,13 @@ def tls_certificate(tls_certificate_authority): @pytest.fixture -def tls_ca_certificate_pem_path(tls_certificate_authority): +def tls_ca_certificate_pem_path(tls_certificate_authority: trustme.CA): with tls_certificate_authority.cert_pem.tempfile() as ca_cert_pem: yield ca_cert_pem @pytest.fixture -def tls_ca_certificate_private_key_path(tls_certificate_authority): +def tls_ca_certificate_private_key_path(tls_certificate_authority: trustme.CA): with tls_certificate_authority.private_key_pem.tempfile() as private_key: yield private_key @@ -49,13 +49,13 @@ def tls_ca_certificate_private_key_encrypted_path(tls_certificate_authority): @pytest.fixture -def tls_certificate_pem_path(tls_certificate): +def tls_certificate_pem_path(tls_certificate: trustme.LeafCert): with tls_certificate.private_key_and_cert_chain_pem.tempfile() as cert_pem: yield cert_pem @pytest.fixture -def tls_ca_ssl_context(tls_certificate): +def tls_ca_ssl_context(tls_certificate: trustme.LeafCert) -> ssl.SSLContext: ssl_ctx = ssl.SSLContext() tls_certificate.configure_cert(ssl_ctx) return ssl_ctx diff --git a/tests/test_ssl.py b/tests/test_ssl.py index a18069374..d44e5de49 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -21,6 +21,7 @@ async def test_run( limit_max_requests=1, ssl_keyfile=tls_ca_certificate_private_key_path, ssl_certfile=tls_ca_certificate_pem_path, + ssl_ca_certs=tls_ca_certificate_pem_path, ) async with run_server(config): async with httpx.AsyncClient(verify=tls_ca_ssl_context) as client: @@ -29,12 +30,15 @@ async def test_run( @pytest.mark.asyncio -async def test_run_chain(tls_ca_ssl_context, tls_certificate_pem_path): +async def test_run_chain( + tls_ca_ssl_context, tls_certificate_pem_path, tls_ca_certificate_pem_path +): config = Config( app=app, loop="asyncio", limit_max_requests=1, ssl_certfile=tls_certificate_pem_path, + ssl_ca_certs=tls_ca_certificate_pem_path, ) async with run_server(config): async with httpx.AsyncClient(verify=tls_ca_ssl_context) as client: @@ -55,6 +59,7 @@ async def test_run_password( ssl_keyfile=tls_ca_certificate_private_key_encrypted_path, ssl_certfile=tls_ca_certificate_pem_path, ssl_keyfile_password="uvicorn password for the win", + ssl_ca_certs=tls_ca_certificate_pem_path, ) async with run_server(config): async with httpx.AsyncClient(verify=tls_ca_ssl_context) as client: From b9a71262ea1f5c020a5a4036c1ea29e55ab4afd6 Mon Sep 17 00:00:00 2001 From: euri10 Date: Tue, 9 Mar 2021 16:11:20 +0100 Subject: [PATCH 128/128] Test with new test run_server --- docs/deployment.md | 2 -- docs/index.md | 1 - tests/protocols/test_websocket.py | 10 +++++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 552b594b3..1402557be 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -47,7 +47,6 @@ Options: $WEB_CONCURRENCY environment variable if available, or 1. Not valid with --reload. - --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] @@ -56,7 +55,6 @@ Options: WebSocket protocol implementation. [default: auto] - --ws-max-size INTEGER WebSocket max size message in bytes [default: 16777216] diff --git a/docs/index.md b/docs/index.md index cebd88996..5b96e22dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -117,7 +117,6 @@ Options: $WEB_CONCURRENCY environment variable if available, or 1. Not valid with --reload. - --loop [auto|asyncio|uvloop] Event loop implementation. [default: auto] --http [auto|h11|httptools] HTTP protocol implementation. [default: auto] diff --git a/tests/protocols/test_websocket.py b/tests/protocols/test_websocket.py index 1428e48da..81c1e922b 100644 --- a/tests/protocols/test_websocket.py +++ b/tests/protocols/test_websocket.py @@ -465,8 +465,9 @@ async def get_subprotocol(url): MAX_WS_BYTES_MINUS1 = 1024 * 1024 * 16 - 1 +@pytest.mark.asyncio @pytest.mark.parametrize("protocol_cls", ONLY_WEBSOCKETPROTOCOL) -def test_send_binary_data_to_server_bigger_than_default(protocol_cls): +async def test_send_binary_data_to_server_bigger_than_default(protocol_cls): class App(WebSocketResponse): async def websocket_connect(self, message): await self.send({"type": "websocket.accept"}) @@ -482,8 +483,7 @@ async def send_text(url): await websocket.send(b"\x01" * DEFAULT_MAX_WS_BYTES_PLUS1) return await websocket.recv() - with run_server(App, protocol_cls=protocol_cls) as url: - loop = asyncio.new_event_loop() - data = loop.run_until_complete(send_text(url)) + config = Config(app=App, ws=protocol_cls, lifespan="off") + async with run_server(config): + data = await send_text("ws://127.0.0.1:8000") assert data == b"\x01" * DEFAULT_MAX_WS_BYTES_PLUS1 - loop.close()