From c11de4e01912b1665f35eb0b56d761ff1ce94b34 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Thu, 26 Oct 2023 21:56:40 +0100 Subject: [PATCH] Fix warning displayed by default (#7677) (#7752) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> (cherry picked from commit 32a1bae0e7513a7708708c41ee4b93fc3c9952cf) --- aiohttp/web.py | 263 ++++++++++++++++++++++----------------------- aiohttp/web_app.py | 255 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 332 insertions(+), 186 deletions(-) diff --git a/aiohttp/web.py b/aiohttp/web.py index 1a30dd87775..3246674d4f5 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -6,8 +6,6 @@ import warnings from argparse import ArgumentParser from collections.abc import Iterable -from contextlib import suppress -from functools import partial from importlib import import_module from typing import ( Any, @@ -21,122 +19,141 @@ Union, cast, ) -from weakref import WeakSet from .abc import AbstractAccessLogger -from .helpers import AppKey +from .helpers import AppKey as AppKey from .log import access_logger from .typedefs import PathLike -from .web_app import Application, CleanupError +from .web_app import Application as Application, CleanupError as CleanupError from .web_exceptions import ( - HTTPAccepted, - HTTPBadGateway, - HTTPBadRequest, - HTTPClientError, - HTTPConflict, - HTTPCreated, - HTTPError, - HTTPException, - HTTPExpectationFailed, - HTTPFailedDependency, - HTTPForbidden, - HTTPFound, - HTTPGatewayTimeout, - HTTPGone, - HTTPInsufficientStorage, - HTTPInternalServerError, - HTTPLengthRequired, - HTTPMethodNotAllowed, - HTTPMisdirectedRequest, - HTTPMove, - HTTPMovedPermanently, - HTTPMultipleChoices, - HTTPNetworkAuthenticationRequired, - HTTPNoContent, - HTTPNonAuthoritativeInformation, - HTTPNotAcceptable, - HTTPNotExtended, - HTTPNotFound, - HTTPNotImplemented, - HTTPNotModified, - HTTPOk, - HTTPPartialContent, - HTTPPaymentRequired, - HTTPPermanentRedirect, - HTTPPreconditionFailed, - HTTPPreconditionRequired, - HTTPProxyAuthenticationRequired, - HTTPRedirection, - HTTPRequestEntityTooLarge, - HTTPRequestHeaderFieldsTooLarge, - HTTPRequestRangeNotSatisfiable, - HTTPRequestTimeout, - HTTPRequestURITooLong, - HTTPResetContent, - HTTPSeeOther, - HTTPServerError, - HTTPServiceUnavailable, - HTTPSuccessful, - HTTPTemporaryRedirect, - HTTPTooManyRequests, - HTTPUnauthorized, - HTTPUnavailableForLegalReasons, - HTTPUnprocessableEntity, - HTTPUnsupportedMediaType, - HTTPUpgradeRequired, - HTTPUseProxy, - HTTPVariantAlsoNegotiates, - HTTPVersionNotSupported, - NotAppKeyWarning, + HTTPAccepted as HTTPAccepted, + HTTPBadGateway as HTTPBadGateway, + HTTPBadRequest as HTTPBadRequest, + HTTPClientError as HTTPClientError, + HTTPConflict as HTTPConflict, + HTTPCreated as HTTPCreated, + HTTPError as HTTPError, + HTTPException as HTTPException, + HTTPExpectationFailed as HTTPExpectationFailed, + HTTPFailedDependency as HTTPFailedDependency, + HTTPForbidden as HTTPForbidden, + HTTPFound as HTTPFound, + HTTPGatewayTimeout as HTTPGatewayTimeout, + HTTPGone as HTTPGone, + HTTPInsufficientStorage as HTTPInsufficientStorage, + HTTPInternalServerError as HTTPInternalServerError, + HTTPLengthRequired as HTTPLengthRequired, + HTTPMethodNotAllowed as HTTPMethodNotAllowed, + HTTPMisdirectedRequest as HTTPMisdirectedRequest, + HTTPMove as HTTPMove, + HTTPMovedPermanently as HTTPMovedPermanently, + HTTPMultipleChoices as HTTPMultipleChoices, + HTTPNetworkAuthenticationRequired as HTTPNetworkAuthenticationRequired, + HTTPNoContent as HTTPNoContent, + HTTPNonAuthoritativeInformation as HTTPNonAuthoritativeInformation, + HTTPNotAcceptable as HTTPNotAcceptable, + HTTPNotExtended as HTTPNotExtended, + HTTPNotFound as HTTPNotFound, + HTTPNotImplemented as HTTPNotImplemented, + HTTPNotModified as HTTPNotModified, + HTTPOk as HTTPOk, + HTTPPartialContent as HTTPPartialContent, + HTTPPaymentRequired as HTTPPaymentRequired, + HTTPPermanentRedirect as HTTPPermanentRedirect, + HTTPPreconditionFailed as HTTPPreconditionFailed, + HTTPPreconditionRequired as HTTPPreconditionRequired, + HTTPProxyAuthenticationRequired as HTTPProxyAuthenticationRequired, + HTTPRedirection as HTTPRedirection, + HTTPRequestEntityTooLarge as HTTPRequestEntityTooLarge, + HTTPRequestHeaderFieldsTooLarge as HTTPRequestHeaderFieldsTooLarge, + HTTPRequestRangeNotSatisfiable as HTTPRequestRangeNotSatisfiable, + HTTPRequestTimeout as HTTPRequestTimeout, + HTTPRequestURITooLong as HTTPRequestURITooLong, + HTTPResetContent as HTTPResetContent, + HTTPSeeOther as HTTPSeeOther, + HTTPServerError as HTTPServerError, + HTTPServiceUnavailable as HTTPServiceUnavailable, + HTTPSuccessful as HTTPSuccessful, + HTTPTemporaryRedirect as HTTPTemporaryRedirect, + HTTPTooManyRequests as HTTPTooManyRequests, + HTTPUnauthorized as HTTPUnauthorized, + HTTPUnavailableForLegalReasons as HTTPUnavailableForLegalReasons, + HTTPUnprocessableEntity as HTTPUnprocessableEntity, + HTTPUnsupportedMediaType as HTTPUnsupportedMediaType, + HTTPUpgradeRequired as HTTPUpgradeRequired, + HTTPUseProxy as HTTPUseProxy, + HTTPVariantAlsoNegotiates as HTTPVariantAlsoNegotiates, + HTTPVersionNotSupported as HTTPVersionNotSupported, + NotAppKeyWarning as NotAppKeyWarning, ) -from .web_fileresponse import FileResponse +from .web_fileresponse import FileResponse as FileResponse from .web_log import AccessLogger -from .web_middlewares import middleware, normalize_path_middleware -from .web_protocol import PayloadAccessError, RequestHandler, RequestPayloadError -from .web_request import BaseRequest, FileField, Request -from .web_response import ContentCoding, Response, StreamResponse, json_response +from .web_middlewares import ( + middleware as middleware, + normalize_path_middleware as normalize_path_middleware, +) +from .web_protocol import ( + PayloadAccessError as PayloadAccessError, + RequestHandler as RequestHandler, + RequestPayloadError as RequestPayloadError, +) +from .web_request import ( + BaseRequest as BaseRequest, + FileField as FileField, + Request as Request, +) +from .web_response import ( + ContentCoding as ContentCoding, + Response as Response, + StreamResponse as StreamResponse, + json_response as json_response, +) from .web_routedef import ( - AbstractRouteDef, - RouteDef, - RouteTableDef, - StaticDef, - delete, - get, - head, - options, - patch, - post, - put, - route, - static, - view, + AbstractRouteDef as AbstractRouteDef, + RouteDef as RouteDef, + RouteTableDef as RouteTableDef, + StaticDef as StaticDef, + delete as delete, + get as get, + head as head, + options as options, + patch as patch, + post as post, + put as put, + route as route, + static as static, + view as view, ) from .web_runner import ( - AppRunner, - BaseRunner, - BaseSite, - GracefulExit, - NamedPipeSite, - ServerRunner, - SockSite, - TCPSite, - UnixSite, + AppRunner as AppRunner, + BaseRunner as BaseRunner, + BaseSite as BaseSite, + GracefulExit as GracefulExit, + NamedPipeSite as NamedPipeSite, + ServerRunner as ServerRunner, + SockSite as SockSite, + TCPSite as TCPSite, + UnixSite as UnixSite, ) -from .web_server import Server +from .web_server import Server as Server from .web_urldispatcher import ( - AbstractResource, - AbstractRoute, - DynamicResource, - PlainResource, - PrefixedSubAppResource, - Resource, - ResourceRoute, - StaticResource, - UrlDispatcher, - UrlMappingMatchInfo, - View, + AbstractResource as AbstractResource, + AbstractRoute as AbstractRoute, + DynamicResource as DynamicResource, + PlainResource as PlainResource, + PrefixedSubAppResource as PrefixedSubAppResource, + Resource as Resource, + ResourceRoute as ResourceRoute, + StaticResource as StaticResource, + UrlDispatcher as UrlDispatcher, + UrlMappingMatchInfo as UrlMappingMatchInfo, + View as View, +) +from .web_ws import ( + WebSocketReady as WebSocketReady, + WebSocketResponse as WebSocketResponse, + WSMsgType as WSMsgType, ) -from .web_ws import WebSocketReady, WebSocketResponse, WSMsgType __all__ = ( # web_app @@ -300,24 +317,7 @@ async def _run_app( reuse_port: Optional[bool] = None, handler_cancellation: bool = False, ) -> None: - async def wait( - starting_tasks: "WeakSet[asyncio.Task[object]]", shutdown_timeout: float - ) -> None: - # Wait for pending tasks for a given time limit. - t = asyncio.current_task() - assert t is not None - starting_tasks.add(t) - with suppress(asyncio.TimeoutError): - await asyncio.wait_for(_wait(starting_tasks), timeout=shutdown_timeout) - - async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: - t = asyncio.current_task() - assert t is not None - exclude.add(t) - while tasks := asyncio.all_tasks().difference(exclude): - await asyncio.wait(tasks) - - # An internal function to actually do all dirty job for application running + # A internal functio to actually do all dirty job for application running if asyncio.iscoroutine(app): app = await app @@ -330,17 +330,10 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: access_log_format=access_log_format, access_log=access_log, keepalive_timeout=keepalive_timeout, - shutdown_timeout=shutdown_timeout, handler_cancellation=handler_cancellation, ) await runner.setup() - # On shutdown we want to avoid waiting on tasks which run forever. - # It's very likely that all tasks which run forever will have been created by - # the time we have completed the application startup (in runner.setup()), - # so we just record all running tasks here and exclude them later. - starting_tasks: "WeakSet[asyncio.Task[object]]" = WeakSet(asyncio.all_tasks()) - runner.shutdown_callback = partial(wait, starting_tasks, shutdown_timeout) sites: List[BaseSite] = [] @@ -352,6 +345,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: runner, host, port, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, reuse_address=reuse_address, @@ -365,6 +359,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: runner, h, port, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, reuse_address=reuse_address, @@ -376,6 +371,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: TCPSite( runner, port=port, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, reuse_address=reuse_address, @@ -389,6 +385,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: UnixSite( runner, path, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) @@ -399,6 +396,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: UnixSite( runner, p, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) @@ -410,6 +408,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: SockSite( runner, sock, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) @@ -420,6 +419,7 @@ async def _wait(exclude: "WeakSet[asyncio.Task[object]]") -> None: SockSite( runner, s, + shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) @@ -468,7 +468,6 @@ def _cancel_tasks( def run_app( app: Union[Application, Awaitable[Application]], *, - debug: bool = False, host: Optional[Union[str, HostSequence]] = None, port: Optional[int] = None, path: Union[PathLike, TypingIterable[PathLike], None] = None, @@ -490,7 +489,6 @@ def run_app( """Run an app locally""" if loop is None: loop = asyncio.new_event_loop() - loop.set_debug(debug) # Configure if and only if in debugging mode and using the default logger if loop.get_debug() and access_log and access_log.name == "aiohttp.access": @@ -531,7 +529,6 @@ def run_app( _cancel_tasks(asyncio.all_tasks(loop), loop) loop.run_until_complete(loop.shutdown_asyncgens()) loop.close() - asyncio.set_event_loop(None) def main(argv: List[str]) -> None: diff --git a/aiohttp/web_app.py b/aiohttp/web_app.py index 355b311ee60..ed5f8ee2636 100644 --- a/aiohttp/web_app.py +++ b/aiohttp/web_app.py @@ -16,11 +16,11 @@ MutableMapping, Optional, Sequence, + Tuple, Type, TypeVar, Union, cast, - final, overload, ) @@ -28,14 +28,25 @@ from frozenlist import FrozenList from . import hdrs -from .helpers import AppKey +from .abc import ( + AbstractAccessLogger, + AbstractMatchInfo, + AbstractRouter, + AbstractStreamWriter, +) +from .helpers import DEBUG, AppKey +from .http_parser import RawRequestMessage from .log import web_logger +from .streams import StreamReader from .typedefs import Middleware from .web_exceptions import NotAppKeyWarning +from .web_log import AccessLogger from .web_middlewares import _fix_request_current_app +from .web_protocol import RequestHandler from .web_request import Request from .web_response import StreamResponse from .web_routedef import AbstractRouteDef +from .web_server import Server from .web_urldispatcher import ( AbstractResource, AbstractRoute, @@ -49,71 +60,86 @@ __all__ = ("Application", "CleanupError") -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover _AppSignal = Signal[Callable[["Application"], Awaitable[None]]] _RespPrepareSignal = Signal[Callable[[Request, StreamResponse], Awaitable[None]]] _Middlewares = FrozenList[Middleware] - _MiddlewaresHandlers = Sequence[Middleware] + _MiddlewaresHandlers = Optional[Sequence[Tuple[Middleware, bool]]] _Subapps = List["Application"] else: # No type checker mode, skip types _AppSignal = Signal _RespPrepareSignal = Signal - _Handler = Callable _Middlewares = FrozenList - _MiddlewaresHandlers = Sequence + _MiddlewaresHandlers = Optional[Sequence] _Subapps = List _T = TypeVar("_T") _U = TypeVar("_U") -@final class Application(MutableMapping[Union[str, AppKey[Any]], Any]): - __slots__ = ( - "logger", - "_debug", - "_router", - "_loop", - "_handler_args", - "_middlewares", - "_middlewares_handlers", - "_run_middlewares", - "_state", - "_frozen", - "_pre_frozen", - "_subapps", - "_on_response_prepare", - "_on_startup", - "_on_shutdown", - "_on_cleanup", - "_client_max_size", - "_cleanup_ctx", + ATTRS = frozenset( + [ + "logger", + "_debug", + "_router", + "_loop", + "_handler_args", + "_middlewares", + "_middlewares_handlers", + "_run_middlewares", + "_state", + "_frozen", + "_pre_frozen", + "_subapps", + "_on_response_prepare", + "_on_startup", + "_on_shutdown", + "_on_cleanup", + "_client_max_size", + "_cleanup_ctx", + ] ) def __init__( self, *, logger: logging.Logger = web_logger, + router: Optional[UrlDispatcher] = None, middlewares: Iterable[Middleware] = (), handler_args: Optional[Mapping[str, Any]] = None, client_max_size: int = 1024**2, + loop: Optional[asyncio.AbstractEventLoop] = None, debug: Any = ..., # mypy doesn't support ellipsis ) -> None: + if router is None: + router = UrlDispatcher() + else: + warnings.warn( + "router argument is deprecated", DeprecationWarning, stacklevel=2 + ) + assert isinstance(router, AbstractRouter), router + + if loop is not None: + warnings.warn( + "loop argument is deprecated", DeprecationWarning, stacklevel=2 + ) + if debug is not ...: warnings.warn( - "debug argument is no-op since 4.0 " "and scheduled for removal in 5.0", - DeprecationWarning, - stacklevel=2, + "debug argument is deprecated", DeprecationWarning, stacklevel=2 ) - self._router = UrlDispatcher() + self._debug = debug + self._router: UrlDispatcher = router + self._loop = loop self._handler_args = handler_args self.logger = logger self._middlewares: _Middlewares = FrozenList(middlewares) # initialized on freezing - self._middlewares_handlers: _MiddlewaresHandlers = tuple() + self._middlewares_handlers: _MiddlewaresHandlers = None # initialized on freezing self._run_middlewares: Optional[bool] = None @@ -132,11 +158,25 @@ def __init__( self._client_max_size = client_max_size def __init_subclass__(cls: Type["Application"]) -> None: - raise TypeError( + warnings.warn( "Inheritance class {} from web.Application " - "is forbidden".format(cls.__name__) + "is discouraged".format(cls.__name__), + DeprecationWarning, + stacklevel=3, ) + if DEBUG: # pragma: no cover + + def __setattr__(self, name: str, val: Any) -> None: + if name not in self.ATTRS: + warnings.warn( + "Setting custom web.Application.{} attribute " + "is discouraged".format(name), + DeprecationWarning, + stacklevel=2, + ) + super().__setattr__(name, val) + # MutableMapping API def __eq__(self, other: object) -> bool: @@ -155,8 +195,10 @@ def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any: def _check_frozen(self) -> None: if self._frozen: - raise RuntimeError( - "Changing state of started or joined " "application is forbidden" + warnings.warn( + "Changing state of started or joined " "application is deprecated", + DeprecationWarning, + stacklevel=3, ) @overload # type: ignore[override] @@ -205,12 +247,31 @@ def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any: return self._state.get(key, default) ######## + @property + def loop(self) -> asyncio.AbstractEventLoop: + # Technically the loop can be None + # but we mask it by explicit type cast + # to provide more convinient type annotation + warnings.warn("loop property is deprecated", DeprecationWarning, stacklevel=2) + return cast(asyncio.AbstractEventLoop, self._loop) + def _set_loop(self, loop: Optional[asyncio.AbstractEventLoop]) -> None: - warnings.warn( - "_set_loop() is no-op since 4.0 " "and scheduled for removal in 5.0", - DeprecationWarning, - stacklevel=2, - ) + if loop is None: + loop = asyncio.get_event_loop() + if self._loop is not None and self._loop is not loop: + raise RuntimeError( + "web.Application instance initialized with different loop" + ) + + self._loop = loop + + # set loop debug + if self._debug is ...: + self._debug = loop.get_debug() + + # set loop to sub applications + for subapp in self._subapps: + subapp._set_loop(loop) @property def pre_frozen(self) -> bool: @@ -256,12 +317,8 @@ def freeze(self) -> None: @property def debug(self) -> bool: - warnings.warn( - "debug property is deprecated since 4.0" "and scheduled for removal in 5.0", - DeprecationWarning, - stacklevel=2, - ) - return asyncio.get_event_loop().get_debug() + warnings.warn("debug property is deprecated", DeprecationWarning, stacklevel=2) + return self._debug # type: ignore[no-any-return] def _reg_subapp_signals(self, subapp: "Application") -> None: def reg_handler(signame: str) -> None: @@ -298,6 +355,8 @@ def _add_subapp( self._reg_subapp_signals(subapp) self._subapps.append(subapp) subapp.pre_freeze() + if self._loop is not None: + subapp._set_loop(self._loop) return resource def add_domain(self, domain: str, subapp: "Application") -> AbstractResource: @@ -341,6 +400,54 @@ def router(self) -> UrlDispatcher: def middlewares(self) -> _Middlewares: return self._middlewares + def _make_handler( + self, + *, + loop: Optional[asyncio.AbstractEventLoop] = None, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + if not issubclass(access_log_class, AbstractAccessLogger): + raise TypeError( + "access_log_class must be subclass of " + "aiohttp.abc.AbstractAccessLogger, got {}".format(access_log_class) + ) + + self._set_loop(loop) + self.freeze() + + kwargs["debug"] = self._debug + kwargs["access_log_class"] = access_log_class + if self._handler_args: + for k, v in self._handler_args.items(): + kwargs[k] = v + + return Server( + self._handle, # type: ignore[arg-type] + request_factory=self._make_request, + loop=self._loop, + **kwargs, + ) + + def make_handler( + self, + *, + loop: Optional[asyncio.AbstractEventLoop] = None, + access_log_class: Type[AbstractAccessLogger] = AccessLogger, + **kwargs: Any, + ) -> Server: + + warnings.warn( + "Application.make_handler(...) is deprecated, " "use AppRunner API instead", + DeprecationWarning, + stacklevel=2, + ) + + return self._make_handler( + loop=loop, access_log_class=access_log_class, **kwargs + ) + async def startup(self) -> None: """Causes on_startup signal @@ -366,13 +473,51 @@ async def cleanup(self) -> None: # If an exception occurs in startup, ensure cleanup contexts are completed. await self._cleanup_ctx._on_cleanup(self) - def _prepare_middleware(self) -> Iterator[Middleware]: - yield from reversed(self._middlewares) - yield _fix_request_current_app(self) + def _make_request( + self, + message: RawRequestMessage, + payload: StreamReader, + protocol: RequestHandler, + writer: AbstractStreamWriter, + task: "asyncio.Task[None]", + _cls: Type[Request] = Request, + ) -> Request: + return _cls( + message, + payload, + protocol, + writer, + task, + self._loop, + client_max_size=self._client_max_size, + ) + + def _prepare_middleware(self) -> Iterator[Tuple[Middleware, bool]]: + for m in reversed(self._middlewares): + if getattr(m, "__middleware_version__", None) == 1: + yield m, True + else: + warnings.warn( + 'old-style middleware "{!r}" deprecated, ' "see #2252".format(m), + DeprecationWarning, + stacklevel=2, + ) + yield m, False + + yield _fix_request_current_app(self), True async def _handle(self, request: Request) -> StreamResponse: + loop = asyncio.get_event_loop() + debug = loop.get_debug() match_info = await self._router.resolve(request) + if debug: # pragma: no cover + if not isinstance(match_info, AbstractMatchInfo): + raise TypeError( + "match_info should be AbstractMatchInfo " + "instance, not {!r}".format(match_info) + ) match_info.add_app(self) + match_info.freeze() resp = None @@ -387,9 +532,13 @@ async def _handle(self, request: Request) -> StreamResponse: if self._run_middlewares: for app in match_info.apps[::-1]: - assert app.pre_frozen, "middleware handlers are not ready" - for m in app._middlewares_handlers: - handler = update_wrapper(partial(m, handler=handler), handler) + for m, new_style in app._middlewares_handlers: # type: ignore[union-attr] + if new_style: + handler = update_wrapper( + partial(m, handler=handler), handler + ) + else: + handler = await m(app, handler) # type: ignore[arg-type,assignment] resp = await handler(request) @@ -412,7 +561,7 @@ def exceptions(self) -> List[BaseException]: return cast(List[BaseException], self.args[1]) -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover _CleanupContextBase = FrozenList[Callable[[Application], AsyncIterator[None]]] else: _CleanupContextBase = FrozenList