diff --git a/docs/concurrent.rst b/docs/concurrent.rst index 5904a60b89..f7a855a38f 100644 --- a/docs/concurrent.rst +++ b/docs/concurrent.rst @@ -11,9 +11,7 @@ .. class:: Future - ``tornado.concurrent.Future`` is an alias for `asyncio.Future` - on Python 3. On Python 2, it provides an equivalent - implementation. + ``tornado.concurrent.Future`` is an alias for `asyncio.Future`. In Tornado, the main way in which applications interact with ``Future`` objects is by ``awaiting`` or ``yielding`` them in diff --git a/docs/conda.yml b/docs/conda.yml index e33288db2a..44c4274a26 100644 --- a/docs/conda.yml +++ b/docs/conda.yml @@ -5,4 +5,5 @@ dependencies: - pip: - sphinx - sphinx-rtd-theme + - sphinxcontrib-asyncio - Twisted diff --git a/docs/conf.py b/docs/conf.py index d6e37faf4f..f0cfa9c292 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,6 +18,7 @@ "sphinx.ext.doctest", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinxcontrib.asyncio", ] primary_domain = "py" @@ -37,6 +38,7 @@ "tornado.platform.asyncio", "tornado.platform.caresresolver", "tornado.platform.twisted", + "tornado.simple_httpclient", ] # I wish this could go in a per-module file... coverage_ignore_classes = [ diff --git a/docs/escape.rst b/docs/escape.rst index 54f1ca9d2d..2a03eddb38 100644 --- a/docs/escape.rst +++ b/docs/escape.rst @@ -17,19 +17,15 @@ Byte/unicode conversions ------------------------ - These functions are used extensively within Tornado itself, - but should not be directly needed by most applications. Note that - much of the complexity of these functions comes from the fact that - Tornado supports both Python 2 and Python 3. .. autofunction:: utf8 .. autofunction:: to_unicode .. function:: native_str + .. function:: to_basestring - Converts a byte or unicode string into type `str`. Equivalent to - `utf8` on Python 2 and `to_unicode` on Python 3. - - .. autofunction:: to_basestring + Converts a byte or unicode string into type `str`. These functions + were used to help transition from Python 2 to Python 3 but are now + deprecated aliases for `to_unicode`. .. autofunction:: recursive_unicode diff --git a/docs/gen.rst b/docs/gen.rst index 320d29c515..4cb5a4f434 100644 --- a/docs/gen.rst +++ b/docs/gen.rst @@ -13,27 +13,27 @@ .. autofunction:: coroutine + .. autoexception:: Return + Utility functions ----------------- - .. autoexception:: Return - - .. autofunction:: with_timeout + .. autofunction:: with_timeout(timeout: Union[float, datetime.timedelta], future: Yieldable, quiet_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = ()) .. autofunction:: sleep - .. autodata:: moment - :annotation: - .. autoclass:: WaitIterator :members: - .. autofunction:: multi + .. autofunction:: multi(Union[List[Yieldable], Dict[Any, Yieldable]], quiet_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = ()) - .. autofunction:: multi_future + .. autofunction:: multi_future(Union[List[Yieldable], Dict[Any, Yieldable]], quiet_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = ()) .. autofunction:: convert_yielded .. autofunction:: maybe_future .. autofunction:: is_coroutine_function + + .. autodata:: moment + :annotation: diff --git a/docs/httpclient.rst b/docs/httpclient.rst index cf3bc8e715..178dc1480a 100644 --- a/docs/httpclient.rst +++ b/docs/httpclient.rst @@ -47,7 +47,9 @@ Implementations ~~~~~~~~~~~~~~~ .. automodule:: tornado.simple_httpclient - :members: + + .. autoclass:: SimpleAsyncHTTPClient + :members: .. module:: tornado.curl_httpclient diff --git a/docs/httpserver.rst b/docs/httpserver.rst index 88c74376bd..ddb7766793 100644 --- a/docs/httpserver.rst +++ b/docs/httpserver.rst @@ -5,5 +5,8 @@ HTTP Server ----------- - .. autoclass:: HTTPServer + .. autoclass:: HTTPServer(request_callback: Union[httputil.HTTPServerConnectionDelegate, Callable[[httputil.HTTPServerRequest], None]], no_keep_alive: bool = False, xheaders: bool = False, ssl_options: Union[Dict[str, Any], ssl.SSLContext] = None, protocol: str = None, decompress_request: bool = False, chunk_size: int = None, max_header_size: int = None, idle_connection_timeout: float = None, body_timeout: float = None, max_body_size: int = None, max_buffer_size: int = None, trusted_downstream: List[str] = None) :members: + + The public interface of this class is mostly inherited from + `.TCPServer` and is documented under that class. diff --git a/docs/ioloop.rst b/docs/ioloop.rst index 55096e1875..5b748d3695 100644 --- a/docs/ioloop.rst +++ b/docs/ioloop.rst @@ -45,10 +45,3 @@ .. automethod:: IOLoop.time .. autoclass:: PeriodicCallback :members: - - Methods for subclasses - ^^^^^^^^^^^^^^^^^^^^^^ - - .. automethod:: IOLoop.initialize - .. automethod:: IOLoop.close_fd - .. automethod:: IOLoop.split_fd diff --git a/docs/locks.rst b/docs/locks.rst index 9f991880c3..df30351cea 100644 --- a/docs/locks.rst +++ b/docs/locks.rst @@ -10,9 +10,10 @@ similar to those provided in the standard library's `asyncio package .. warning:: - Note that these primitives are not actually thread-safe and cannot be used in - place of those from the standard library--they are meant to coordinate Tornado - coroutines in a single-threaded app, not to protect shared objects in a + Note that these primitives are not actually thread-safe and cannot + be used in place of those from the standard library's `threading` + module--they are meant to coordinate Tornado coroutines in a + single-threaded app, not to protect shared objects in a multithreaded app. .. automodule:: tornado.locks diff --git a/docs/releases/v4.2.0.rst b/docs/releases/v4.2.0.rst index 93493ee113..bacfb13a05 100644 --- a/docs/releases/v4.2.0.rst +++ b/docs/releases/v4.2.0.rst @@ -162,7 +162,7 @@ Then the Tornado equivalent is:: * The `.IOLoop` constructor now has a ``make_current`` keyword argument to control whether the new `.IOLoop` becomes `.IOLoop.current()`. * Third-party implementations of `.IOLoop` should accept ``**kwargs`` - in their `~.IOLoop.initialize` methods and pass them to the superclass + in their ``IOLoop.initialize`` methods and pass them to the superclass implementation. * `.PeriodicCallback` is now more efficient when the clock jumps forward by a large amount. diff --git a/docs/requirements.txt b/docs/requirements.txt index 69c93b12cc..97528d9708 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ sphinx>1.8.2 +sphinxcontrib-asyncio sphinx_rtd_theme Twisted diff --git a/docs/web.rst b/docs/web.rst index ed9a3f1c1b..fbf2e826ef 100644 --- a/docs/web.rst +++ b/docs/web.rst @@ -9,7 +9,7 @@ Request handlers ---------------- - .. autoclass:: RequestHandler + .. autoclass:: RequestHandler(...) Entry points ^^^^^^^^^^^^ @@ -50,11 +50,24 @@ Input ^^^^^ - .. automethod:: RequestHandler.get_argument + The ``argument`` methods provide support for HTML form-style + arguments. These methods are available in both singular and plural + forms because HTML forms are ambiguous and do not distinguish + between a singular argument and a list containing one entry. If you + wish to use other formats for arguments (for example, JSON), parse + ``self.request.body`` yourself:: + + def prepare(self): + if self.request.headers['Content-Type'] == 'application/x-json': + self.args = json_decode(self.request.body) + # Access self.args directly instead of using self.get_argument. + + + .. automethod:: RequestHandler.get_argument(name: str, default: Union[None, str, RAISE] = RAISE, strip: bool = True) -> Optional[str] .. automethod:: RequestHandler.get_arguments - .. automethod:: RequestHandler.get_query_argument + .. automethod:: RequestHandler.get_query_argument(name: str, default: Union[None, str, RAISE] = RAISE, strip: bool = True) -> Optional[str] .. automethod:: RequestHandler.get_query_arguments - .. automethod:: RequestHandler.get_body_argument + .. automethod:: RequestHandler.get_body_argument(name: str, default: Union[None, str, RAISE] = RAISE, strip: bool = True) -> Optional[str] .. automethod:: RequestHandler.get_body_arguments .. automethod:: RequestHandler.decode_argument .. attribute:: RequestHandler.request @@ -147,8 +160,7 @@ Application configuration ----------------------------- - .. autoclass:: Application - :members: + .. autoclass:: Application(handlers: List[Union[Rule, Tuple]] = None, default_host: str = None, transforms: List[Type[OutputTransform]] = None, **settings) .. attribute:: settings @@ -266,6 +278,12 @@ should be a dictionary of keyword arguments to be passed to the handler's ``initialize`` method. + .. automethod:: Application.listen + .. automethod:: Application.add_handlers(handlers: List[Union[Rule, Tuple]]) + .. automethod:: Application.get_handler_delegate + .. automethod:: Application.reverse_url + .. automethod:: Application.log_request + .. autoclass:: URLSpec The ``URLSpec`` class is also available under the name ``tornado.web.url``. diff --git a/tornado/auth.py b/tornado/auth.py index 115c009231..3757a15487 100644 --- a/tornado/auth.py +++ b/tornado/auth.py @@ -102,7 +102,7 @@ def authenticate_redirect( .. versionchanged:: 6.0 - The ``callback`` argument was removed and this method no + The ``callback`` argument was removed and this method no longer returns an awaitable object. It is now an ordinary synchronous function. """ @@ -128,7 +128,7 @@ async def get_authenticated_user( .. versionchanged:: 6.0 - The ``callback`` argument was removed. Use the returned + The ``callback`` argument was removed. Use the returned awaitable object instead. """ handler = cast(RequestHandler, self) diff --git a/tornado/concurrent.py b/tornado/concurrent.py index df12da87f0..b7bb15c018 100644 --- a/tornado/concurrent.py +++ b/tornado/concurrent.py @@ -14,17 +14,15 @@ # under the License. """Utilities for working with ``Future`` objects. -``Futures`` are a pattern for concurrent programming introduced in -Python 3.2 in the `concurrent.futures` package, and also adopted (in a -slightly different form) in Python 3.4's `asyncio` package. This -package defines a ``Future`` class that is an alias for `asyncio.Future` -when available, and a compatible implementation for older versions of -Python. It also includes some utility functions for interacting with -``Future`` objects. - -While this package is an important part of Tornado's internal +Tornado previously provided its own ``Future`` class, but now uses +`asyncio.Future`. This module contains utility functions for working +with `asyncio.Future` in a way that is backwards-compatible with +Tornado's old ``Future`` implementation. + +While this module is an important part of Tornado's internal implementation, applications rarely need to interact with it directly. + """ import asyncio diff --git a/tornado/escape.py b/tornado/escape.py index bd73e305bf..b0ec332301 100644 --- a/tornado/escape.py +++ b/tornado/escape.py @@ -24,7 +24,7 @@ import re import urllib.parse -from tornado.util import unicode_type, basestring_type +from tornado.util import unicode_type import typing from typing import Union, Any, Optional, Dict, List, Callable @@ -76,7 +76,10 @@ def json_encode(value: Any) -> str: def json_decode(value: Union[str, bytes]) -> Any: - """Returns Python objects for the given JSON string.""" + """Returns Python objects for the given JSON string. + + Supports both `str` and `bytes` inputs. + """ return json.loads(to_basestring(value)) @@ -231,39 +234,7 @@ def to_unicode(value: Union[None, str, bytes]) -> Optional[str]: # noqa: F811 # When dealing with the standard library across python 2 and 3 it is # sometimes useful to have a direct conversion to the native string type native_str = to_unicode - -_BASESTRING_TYPES = (basestring_type, type(None)) - - -@typing.overload -def to_basestring(value: str) -> str: - pass - - -@typing.overload # noqa: F811 -def to_basestring(value: bytes) -> str: - pass - - -@typing.overload # noqa: F811 -def to_basestring(value: None) -> None: - pass - - -def to_basestring(value: Union[None, str, bytes]) -> Optional[str]: # noqa: F811 - """Converts a string argument to a subclass of basestring. - - In python2, byte and unicode strings are mostly interchangeable, - so functions that deal with a user-supplied argument in combination - with ascii string constants can use either and should return the type - the user supplied. In python3, the two types are not interchangeable, - so this method is needed to convert byte strings to unicode. - """ - if isinstance(value, _BASESTRING_TYPES): - return value - if not isinstance(value, bytes): - raise TypeError("Expected bytes, unicode, or None; got %r" % type(value)) - return value.decode("utf-8") +to_basestring = to_unicode def recursive_unicode(obj: Any) -> Any: diff --git a/tornado/gen.py b/tornado/gen.py index e2c77d998a..3c1deeae3d 100644 --- a/tornado/gen.py +++ b/tornado/gen.py @@ -32,12 +32,12 @@ def get(self): .. testoutput:: :hide: -Most asynchronous functions in Tornado return a `.Future`; -yielding this object returns its ``Future.result``. +Asynchronous functions in Tornado return an ``Awaitable`` or `.Future`; +yielding this object returns its result. -You can also yield a list or dict of ``Futures``, which will be -started at the same time and run in parallel; a list or dict of results will -be returned when they are all finished: +You can also yield a list or dict of other yieldable objects, which +will be started at the same time and run in parallel; a list or dict +of results will be returned when they are all finished: .. testcode:: @@ -54,13 +54,9 @@ def get(self): .. testoutput:: :hide: -If the `~functools.singledispatch` library is available (standard in -Python 3.4, available via the `singledispatch -`_ package on older -versions), additional types of objects may be yielded. Tornado includes -support for ``asyncio.Future`` and Twisted's ``Deferred`` class when -``tornado.platform.asyncio`` and ``tornado.platform.twisted`` are imported. -See the `convert_yielded` function to extend this mechanism. +If ``tornado.platform.twisted`` is imported, it is also possible to +yield Twisted's ``Deferred`` objects. See the `convert_yielded` +function to extend this mechanism. .. versionchanged:: 3.2 Dict support added. @@ -162,16 +158,9 @@ def coroutine( ) -> Callable[..., "Future[_T]"]: """Decorator for asynchronous generators. - Any generator that yields objects from this module must be wrapped - in this decorator (or use ``async def`` and ``await`` for similar - functionality). - - Coroutines may "return" by raising the special exception - `Return(value) `. In Python 3.3+, it is also possible for - the function to simply use the ``return value`` statement (prior to - Python 3.3 generators were not allowed to also return values). - In all versions of Python a coroutine that simply wishes to exit - early may use the ``return`` statement without a value. + For compatibility with older versions of Python, coroutines may + also "return" by raising the special exception `Return(value) + `. Functions with this decorator return a `.Future`. @@ -294,22 +283,23 @@ def __init__(self, value: Any = None) -> None: class WaitIterator(object): - """Provides an iterator to yield the results of futures as they finish. + """Provides an iterator to yield the results of awaitables as they finish. - Yielding a set of futures like this: + Yielding a set of awaitables like this: - ``results = yield [future1, future2]`` + ``results = yield [awaitable1, awaitable2]`` - pauses the coroutine until both ``future1`` and ``future2`` + pauses the coroutine until both ``awaitable1`` and ``awaitable2`` return, and then restarts the coroutine with the results of both - futures. If either future is an exception, the expression will - raise that exception and all the results will be lost. + awaitables. If either awaitable raises an exception, the + expression will raise that exception and all the results will be + lost. - If you need to get the result of each future as soon as possible, - or if you need the result of some futures even if others produce + If you need to get the result of each awaitable as soon as possible, + or if you need the result of some awaitables even if others produce errors, you can use ``WaitIterator``:: - wait_iterator = gen.WaitIterator(future1, future2) + wait_iterator = gen.WaitIterator(awaitable1, awaitable2) while not wait_iterator.done(): try: result = yield wait_iterator.next() @@ -325,7 +315,7 @@ class WaitIterator(object): input arguments*. If you need to know which future produced the current result, you can use the attributes ``WaitIterator.current_future``, or ``WaitIterator.current_index`` - to get the index of the future from the input list. (if keyword + to get the index of the awaitable from the input list. (if keyword arguments were used in the construction of the `WaitIterator`, ``current_index`` will use the corresponding keyword). @@ -681,6 +671,9 @@ def done(self) -> bool: Usage: ``yield gen.moment`` +In native coroutines, the equivalent of ``yield gen.moment`` is +``await asyncio.sleep(0)``. + .. versionadded:: 4.0 .. deprecated:: 4.5 @@ -815,7 +808,9 @@ def handle_exception( def convert_yielded(yielded: _Yieldable) -> Future: """Convert a yielded object into a `.Future`. - The default implementation accepts lists, dictionaries, and Futures. + The default implementation accepts lists, dictionaries, and + Futures. This has the side effect of starting any coroutines that + did not start themselves, similar to `asyncio.ensure_future`. If the `~functools.singledispatch` library is available, this function may be extended to support additional types. For example:: @@ -825,6 +820,7 @@ def _(asyncio_future): return tornado.platform.asyncio.to_tornado_future(asyncio_future) .. versionadded:: 4.1 + """ if yielded is None or yielded is moment: return moment diff --git a/tornado/httpclient.py b/tornado/httpclient.py index 8606c01e60..e02c60e3d6 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -20,8 +20,6 @@ * ``curl_httpclient`` is faster. -* ``curl_httpclient`` was the default prior to Tornado 2.0. - Note that if you are using ``curl_httpclient``, it is highly recommended that you use a recent version of ``libcurl`` and ``pycurl``. Currently the minimum supported version of libcurl is @@ -55,7 +53,7 @@ from tornado.ioloop import IOLoop from tornado.util import Configurable -from typing import Type, Any, Union, Dict, Callable, Optional, cast +from typing import Type, Any, Union, Dict, Callable, Optional, cast, Awaitable class HTTPClient(object): @@ -251,7 +249,7 @@ def fetch( request: Union[str, "HTTPRequest"], raise_error: bool = True, **kwargs: Any - ) -> "Future[HTTPResponse]": + ) -> Awaitable["HTTPResponse"]: """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. diff --git a/tornado/httpserver.py b/tornado/httpserver.py index 9376c56859..8044a4f828 100644 --- a/tornado/httpserver.py +++ b/tornado/httpserver.py @@ -168,6 +168,10 @@ def initialize( max_buffer_size: int = None, trusted_downstream: List[str] = None, ) -> None: + # This method's signature is not extracted with autodoc + # because we want its arguments to appear on the class + # constructor. When changing this signature, also update the + # copy in httpserver.rst. self.request_callback = request_callback self.xheaders = xheaders self.protocol = protocol @@ -198,6 +202,19 @@ def configurable_default(cls) -> Type[Configurable]: return HTTPServer async def close_all_connections(self) -> None: + """Close all open connections and asynchronously wait for them to finish. + + This method is used in combination with `~.TCPServer.stop` to + support clean shutdowns (especially for unittests). Typical + usage would call ``stop()`` first to stop accepting new + connections, then ``await close_all_connections()`` to wait for + existing connections to finish. + + This method does not currently close open websocket connections. + + Note that this method is a coroutine and must be caled with ``await``. + + """ while self._connections: # Peek at an arbitrary element of the set conn = next(iter(self._connections)) diff --git a/tornado/ioloop.py b/tornado/ioloop.py index 7cecb01b85..14f73c83ee 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -15,7 +15,11 @@ """An I/O event loop for non-blocking sockets. -On Python 3, `.IOLoop` is a wrapper around the `asyncio` event loop. +In Tornado 6.0, `.IOLoop` is a wrapper around the `asyncio` event +loop, with a slightly different interface for historical reasons. +Applications can use either the `.IOLoop` interface or the underlying +`asyncio` event loop directly (unless compatibility with older +versions of Tornado is desired, in which case `.IOLoop` must be used). Typical applications will use a single `IOLoop` object, accessed via `IOLoop.current` class method. The `IOLoop.start` method (or @@ -24,10 +28,6 @@ may use more than one `IOLoop`, such as one `IOLoop` per thread, or per `unittest` case. -In addition to I/O events, the `IOLoop` can also schedule time-based -events. `IOLoop.add_timeout` is a non-blocking alternative to -`time.sleep`. - """ import asyncio @@ -75,14 +75,10 @@ def close(self) -> None: class IOLoop(Configurable): - """A level-triggered I/O loop. + """An I/O event loop. - On Python 3, `IOLoop` is a wrapper around the `asyncio` event - loop. On Python 2, it uses ``epoll`` (Linux) or ``kqueue`` (BSD - and Mac OS X) if they are available, or else we fall back on - select(). If you are implementing a system that needs to handle - thousands of simultaneous connections, you should use a system - that supports either ``epoll`` or ``kqueue``. + As of Tornado 6.0, `IOLoop` is a wrapper around the `asyncio` event + loop. Example usage for a simple TCP server: @@ -540,12 +536,11 @@ def time(self) -> float: The return value is a floating-point number relative to an unspecified time in the past. - By default, the `IOLoop`'s time function is `time.time`. However, - it may be configured to use e.g. `time.monotonic` instead. - Calls to `add_timeout` that pass a number instead of a - `datetime.timedelta` should use this function to compute the - appropriate time, so they can work no matter what time function - is chosen. + Historically, the IOLoop could be customized to use e.g. + `time.monotonic` instead of `time.time`, but this is not + currently supported and so this method is equivalent to + `time.time`. + """ return time.time() @@ -758,37 +753,37 @@ def _discard_future_result(self, future: Future) -> None: def split_fd( self, fd: Union[int, _Selectable] ) -> Tuple[int, Union[int, _Selectable]]: - """Returns an (fd, obj) pair from an ``fd`` parameter. + # """Returns an (fd, obj) pair from an ``fd`` parameter. - We accept both raw file descriptors and file-like objects as - input to `add_handler` and related methods. When a file-like - object is passed, we must retain the object itself so we can - close it correctly when the `IOLoop` shuts down, but the - poller interfaces favor file descriptors (they will accept - file-like objects and call ``fileno()`` for you, but they - always return the descriptor itself). + # We accept both raw file descriptors and file-like objects as + # input to `add_handler` and related methods. When a file-like + # object is passed, we must retain the object itself so we can + # close it correctly when the `IOLoop` shuts down, but the + # poller interfaces favor file descriptors (they will accept + # file-like objects and call ``fileno()`` for you, but they + # always return the descriptor itself). - This method is provided for use by `IOLoop` subclasses and should - not generally be used by application code. + # This method is provided for use by `IOLoop` subclasses and should + # not generally be used by application code. - .. versionadded:: 4.0 - """ + # .. versionadded:: 4.0 + # """ if isinstance(fd, int): return fd, fd return fd.fileno(), fd def close_fd(self, fd: Union[int, _Selectable]) -> None: - """Utility method to close an ``fd``. + # """Utility method to close an ``fd``. - If ``fd`` is a file-like object, we close it directly; otherwise - we use `os.close`. + # If ``fd`` is a file-like object, we close it directly; otherwise + # we use `os.close`. - This method is provided for use by `IOLoop` subclasses (in - implementations of ``IOLoop.close(all_fds=True)`` and should - not generally be used by application code. + # This method is provided for use by `IOLoop` subclasses (in + # implementations of ``IOLoop.close(all_fds=True)`` and should + # not generally be used by application code. - .. versionadded:: 4.0 - """ + # .. versionadded:: 4.0 + # """ try: if isinstance(fd, int): os.close(fd) diff --git a/tornado/iostream.py b/tornado/iostream.py index a4025d35fb..a9bf977bd7 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -231,9 +231,9 @@ class BaseIOStream(object): """A utility class to write to and read from a non-blocking file or socket. We support a non-blocking ``write()`` and a family of ``read_*()`` - methods. When the operation completes, the `.Future` will resolve + methods. When the operation completes, the ``Awaitable`` will resolve with the data read (or ``None`` for ``write()``). All outstanding - ``Futures`` will resolve with a `StreamClosedError` when the + ``Awaitables`` will resolve with a `StreamClosedError` when the stream is closed; `.BaseIOStream.set_close_callback` can also be used to be notified of a closed stream. diff --git a/tornado/locks.py b/tornado/locks.py index 0da19766c4..8b1a534109 100644 --- a/tornado/locks.py +++ b/tornado/locks.py @@ -20,7 +20,7 @@ from tornado import gen, ioloop from tornado.concurrent import Future, future_set_result_unless_cancelled -from typing import Union, Optional, Type, Any +from typing import Union, Optional, Type, Any, Awaitable import typing if typing.TYPE_CHECKING: @@ -121,7 +121,7 @@ def __repr__(self) -> str: result += " waiters[%s]" % len(self._waiters) return result + ">" - def wait(self, timeout: Union[float, datetime.timedelta] = None) -> "Future[bool]": + def wait(self, timeout: Union[float, datetime.timedelta] = None) -> Awaitable[bool]: """Wait for `.notify`. Returns a `.Future` that resolves ``True`` if the condition is notified, @@ -231,10 +231,10 @@ def clear(self) -> None: """ self._value = False - def wait(self, timeout: Union[float, datetime.timedelta] = None) -> "Future[None]": + def wait(self, timeout: Union[float, datetime.timedelta] = None) -> Awaitable[None]: """Block until the internal flag is true. - Returns a Future, which raises `tornado.util.TimeoutError` after a + Returns an awaitable, which raises `tornado.util.TimeoutError` after a timeout. """ fut = Future() # type: Future[None] @@ -413,10 +413,10 @@ def release(self) -> None: def acquire( self, timeout: Union[float, datetime.timedelta] = None - ) -> "Future[_ReleasingContextManager]": - """Decrement the counter. Returns a Future. + ) -> Awaitable[_ReleasingContextManager]: + """Decrement the counter. Returns an awaitable. - Block if the counter is zero and wait for a `.release`. The Future + Block if the counter is zero and wait for a `.release`. The awaitable raises `.TimeoutError` after the deadline. """ waiter = Future() # type: Future[_ReleasingContextManager] @@ -440,10 +440,7 @@ def on_timeout() -> None: return waiter def __enter__(self) -> None: - raise RuntimeError( - "Use Semaphore like 'with (yield semaphore.acquire())', not like" - " 'with semaphore'" - ) + raise RuntimeError("Use 'async with' instead of 'with' for Semaphore") def __exit__( self, @@ -530,10 +527,10 @@ def __repr__(self) -> str: def acquire( self, timeout: Union[float, datetime.timedelta] = None - ) -> "Future[_ReleasingContextManager]": - """Attempt to lock. Returns a Future. + ) -> Awaitable[_ReleasingContextManager]: + """Attempt to lock. Returns an awaitable. - Returns a Future, which raises `tornado.util.TimeoutError` after a + Returns an awaitable, which raises `tornado.util.TimeoutError` after a timeout. """ return self._block.acquire(timeout) @@ -551,7 +548,7 @@ def release(self) -> None: raise RuntimeError("release unlocked lock") def __enter__(self) -> None: - raise RuntimeError("Use Lock like 'with (yield lock)', not like 'with lock'") + raise RuntimeError("Use `async with` instead of `with` for Lock") def __exit__( self, diff --git a/tornado/queues.py b/tornado/queues.py index 6c4fbd7556..d135c3cb01 100644 --- a/tornado/queues.py +++ b/tornado/queues.py @@ -222,10 +222,10 @@ def put_nowait(self, item: _T) -> None: else: self.__put_internal(item) - def get(self, timeout: Union[float, datetime.timedelta] = None) -> "Future[_T]": + def get(self, timeout: Union[float, datetime.timedelta] = None) -> Awaitable[_T]: """Remove and return an item from the queue. - Returns a Future which resolves once an item is available, or raises + Returns an awaitable which resolves once an item is available, or raises `tornado.util.TimeoutError` after a timeout. ``timeout`` may be a number denoting a time (on the same @@ -277,10 +277,10 @@ def task_done(self) -> None: if self._unfinished_tasks == 0: self._finished.set() - def join(self, timeout: Union[float, datetime.timedelta] = None) -> "Future[None]": + def join(self, timeout: Union[float, datetime.timedelta] = None) -> Awaitable[None]: """Block until all items in the queue are processed. - Returns a Future, which raises `tornado.util.TimeoutError` after a + Returns an awaitable, which raises `tornado.util.TimeoutError` after a timeout. """ return self._finished.wait(timeout) diff --git a/tornado/test/locks_test.py b/tornado/test/locks_test.py index 787bfca314..7f88ab34b6 100644 --- a/tornado/test/locks_test.py +++ b/tornado/test/locks_test.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import asyncio from datetime import timedelta import typing # noqa: F401 import unittest @@ -164,7 +165,7 @@ def test_nested_notify(self): c = locks.Condition() # Three waiters. - futures = [c.wait() for _ in range(3)] + futures = [asyncio.ensure_future(c.wait()) for _ in range(3)] # First and second futures resolved. Second future reenters notify(), # resolving third future. @@ -180,7 +181,7 @@ def test_garbage_collection(self): for _ in range(101): c.wait(timedelta(seconds=0.01)) - future = c.wait() + future = asyncio.ensure_future(c.wait()) self.assertEqual(102, len(c._waiters)) # Let first 101 waiters time out, triggering a collection. @@ -204,11 +205,11 @@ def test_repr(self): def test_event(self): e = locks.Event() - future_0 = e.wait() + future_0 = asyncio.ensure_future(e.wait()) e.set() - future_1 = e.wait() + future_1 = asyncio.ensure_future(e.wait()) e.clear() - future_2 = e.wait() + future_2 = asyncio.ensure_future(e.wait()) self.assertTrue(future_0.done()) self.assertTrue(future_1.done()) @@ -232,9 +233,9 @@ def test_event_set_multiple(self): def test_event_wait_clear(self): e = locks.Event() - f0 = e.wait() + f0 = asyncio.ensure_future(e.wait()) e.clear() - f1 = e.wait() + f1 = asyncio.ensure_future(e.wait()) e.set() self.assertTrue(f0.done()) self.assertTrue(f1.done()) @@ -256,13 +257,13 @@ def test_repr(self): def test_acquire(self): sem = locks.Semaphore() - f0 = sem.acquire() + f0 = asyncio.ensure_future(sem.acquire()) self.assertTrue(f0.done()) # Wait for release(). - f1 = sem.acquire() + f1 = asyncio.ensure_future(sem.acquire()) self.assertFalse(f1.done()) - f2 = sem.acquire() + f2 = asyncio.ensure_future(sem.acquire()) sem.release() self.assertTrue(f1.done()) self.assertFalse(f2.done()) @@ -271,7 +272,7 @@ def test_acquire(self): sem.release() # Now acquire() is instant. - self.assertTrue(sem.acquire().done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) self.assertEqual(0, len(sem._waiters)) @gen_test @@ -286,7 +287,7 @@ def test_acquire_timeout(self): yield acquire sem.acquire() - f = sem.acquire() + f = asyncio.ensure_future(sem.acquire()) self.assertFalse(f.done()) sem.release() self.assertTrue(f.done()) @@ -309,18 +310,21 @@ def test_release_unacquired(self): sem.release() # Now the counter is 3. We can acquire three times before blocking. - self.assertTrue(sem.acquire().done()) - self.assertTrue(sem.acquire().done()) - self.assertTrue(sem.acquire().done()) - self.assertFalse(sem.acquire().done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) + self.assertFalse(asyncio.ensure_future(sem.acquire()).done()) @gen_test def test_garbage_collection(self): # Test that timed-out waiters are occasionally cleaned from the queue. sem = locks.Semaphore(value=0) - futures = [sem.acquire(timedelta(seconds=0.01)) for _ in range(101)] + futures = [ + asyncio.ensure_future(sem.acquire(timedelta(seconds=0.01))) + for _ in range(101) + ] - future = sem.acquire() + future = asyncio.ensure_future(sem.acquire()) self.assertEqual(102, len(sem._waiters)) # Let first 101 waiters time out, triggering a collection. @@ -345,7 +349,7 @@ def test_context_manager(self): self.assertTrue(yielded is None) # Semaphore was released and can be acquired again. - self.assertTrue(sem.acquire().done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) @gen_test def test_context_manager_async_await(self): @@ -359,7 +363,7 @@ async def f(): yield f() # Semaphore was released and can be acquired again. - self.assertTrue(sem.acquire().done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) @gen_test def test_context_manager_exception(self): @@ -369,7 +373,7 @@ def test_context_manager_exception(self): 1 / 0 # Semaphore was released and can be acquired again. - self.assertTrue(sem.acquire().done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) @gen_test def test_context_manager_timeout(self): @@ -378,7 +382,7 @@ def test_context_manager_timeout(self): pass # Semaphore was released and can be acquired again. - self.assertTrue(sem.acquire().done()) + self.assertTrue(asyncio.ensure_future(sem.acquire()).done()) @gen_test def test_context_manager_timeout_error(self): @@ -388,7 +392,7 @@ def test_context_manager_timeout_error(self): pass # Counter is still 0. - self.assertFalse(sem.acquire().done()) + self.assertFalse(asyncio.ensure_future(sem.acquire()).done()) @gen_test def test_context_manager_contended(self): @@ -433,7 +437,7 @@ def test_release_unacquired(self): # Value is 0. sem.acquire() # Block on acquire(). - future = sem.acquire() + future = asyncio.ensure_future(sem.acquire()) self.assertFalse(future.done()) sem.release() self.assertTrue(future.done()) @@ -452,8 +456,8 @@ def test_repr(self): def test_acquire_release(self): lock = locks.Lock() - self.assertTrue(lock.acquire().done()) - future = lock.acquire() + self.assertTrue(asyncio.ensure_future(lock.acquire()).done()) + future = asyncio.ensure_future(lock.acquire()) self.assertFalse(future.done()) lock.release() self.assertTrue(future.done()) @@ -461,7 +465,7 @@ def test_acquire_release(self): @gen_test def test_acquire_fifo(self): lock = locks.Lock() - self.assertTrue(lock.acquire().done()) + self.assertTrue(asyncio.ensure_future(lock.acquire()).done()) N = 5 history = [] @@ -481,7 +485,7 @@ def test_acquire_fifo_async_with(self): # Repeat the above test using `async with lock:` # instead of `with (yield lock.acquire()):`. lock = locks.Lock() - self.assertTrue(lock.acquire().done()) + self.assertTrue(asyncio.ensure_future(lock.acquire()).done()) N = 5 history = [] @@ -502,7 +506,7 @@ def test_acquire_timeout(self): yield lock.acquire(timeout=timedelta(seconds=0.01)) # Still locked. - self.assertFalse(lock.acquire().done()) + self.assertFalse(asyncio.ensure_future(lock.acquire()).done()) def test_multi_release(self): lock = locks.Lock() diff --git a/tornado/test/queues_test.py b/tornado/test/queues_test.py index c46c0585ba..8e527dbb30 100644 --- a/tornado/test/queues_test.py +++ b/tornado/test/queues_test.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import asyncio from datetime import timedelta from random import random import unittest @@ -143,8 +144,10 @@ def test_get_clears_timed_out_putters(self): @gen_test def test_get_clears_timed_out_getters(self): q = queues.Queue() # type: queues.Queue[int] - getters = [q.get(timedelta(seconds=0.01)) for _ in range(10)] - get = q.get() + getters = [ + asyncio.ensure_future(q.get(timedelta(seconds=0.01))) for _ in range(10) + ] + get = asyncio.ensure_future(q.get()) self.assertEqual(11, len(q._getters)) yield gen.sleep(0.02) self.assertEqual(11, len(q._getters)) @@ -259,8 +262,10 @@ def test_put_clears_timed_out_putters(self): @gen_test def test_put_clears_timed_out_getters(self): q = queues.Queue() # type: queues.Queue[int] - getters = [q.get(timedelta(seconds=0.01)) for _ in range(10)] - get = q.get() + getters = [ + asyncio.ensure_future(q.get(timedelta(seconds=0.01))) for _ in range(10) + ] + get = asyncio.ensure_future(q.get()) q.get() self.assertEqual(12, len(q._getters)) yield gen.sleep(0.02) diff --git a/tornado/test/twisted_test.py b/tornado/test/twisted_test.py index 06c49e3b4d..0e03cec9f4 100644 --- a/tornado/test/twisted_test.py +++ b/tornado/test/twisted_test.py @@ -13,10 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Unittest for the twisted-style reactor. -""" - +import asyncio import logging import signal import unittest @@ -120,7 +117,7 @@ def run_reactor(self): def tornado_fetch(self, url, runner): client = AsyncHTTPClient() - fut = client.fetch(url) + fut = asyncio.ensure_future(client.fetch(url)) fut.add_done_callback(lambda f: self.stop_loop()) runner() return fut.result() diff --git a/tornado/web.py b/tornado/web.py index 1f4a169824..8207ccab7e 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -180,6 +180,11 @@ class RequestHandler(object): Subclasses must define at least one of the methods defined in the "Entry points" section below. + + Applications should not construct `RequestHandler` objects + directly and subclasses should not override ``__init__`` (override + `~RequestHandler.initialize` instead). + """ SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PATCH", "PUT", "OPTIONS") @@ -272,10 +277,10 @@ def prepare(self) -> Optional[Awaitable[None]]: Override this method to perform common initialization regardless of the request method. - Asynchronous support: Decorate this method with `.gen.coroutine` - or use ``async def`` to make it asynchronous. - If this method returns a `.Future` execution will not proceed - until the `.Future` is done. + Asynchronous support: Use ``async def`` or decorate this method with + `.gen.coroutine` to make it asynchronous. + If this method returns an ``Awaitable`` execution will not proceed + until the ``Awaitable`` is done. .. versionadded:: 3.1 Asynchronous support. @@ -361,9 +366,10 @@ def get_status(self) -> int: def set_header(self, name: str, value: _HeaderTypes) -> None: """Sets the given response header name and value. - If a datetime is given, we automatically format it according to the - HTTP specification. If the value is not a string, we convert it to - a string. All header values are then encoded as UTF-8. + All header values are converted to strings (`datetime` objects + are formatted according to the HTTP specification for the + ``Date`` header). + """ self._headers[name] = self._convert_header_value(value) @@ -441,10 +447,10 @@ def get_argument( # noqa: F811 If default is not provided, the argument is considered to be required, and we raise a `MissingArgumentError` if it is missing. - If the argument appears in the url more than once, we return the + If the argument appears in the request more than once, we return the last value. - The returned value is always unicode. + This method searches both the query and body arguments. """ return self._get_argument(name, default, self.request.arguments, strip) @@ -453,7 +459,7 @@ def get_arguments(self, name: str, strip: bool = True) -> List[str]: If the argument is not present, returns an empty list. - The returned values are always unicode. + This method searches both the query and body arguments. """ # Make sure `get_arguments` isn't accidentally being called with a @@ -478,8 +484,6 @@ def get_body_argument( If the argument appears in the url more than once, we return the last value. - The returned value is always unicode. - .. versionadded:: 3.2 """ return self._get_argument(name, default, self.request.body_arguments, strip) @@ -489,8 +493,6 @@ def get_body_arguments(self, name: str, strip: bool = True) -> List[str]: If the argument is not present, returns an empty list. - The returned values are always unicode. - .. versionadded:: 3.2 """ return self._get_arguments(name, self.request.body_arguments, strip) @@ -510,8 +512,6 @@ def get_query_argument( If the argument appears in the url more than once, we return the last value. - The returned value is always unicode. - .. versionadded:: 3.2 """ return self._get_argument(name, default, self.request.query_arguments, strip) @@ -521,8 +521,6 @@ def get_query_arguments(self, name: str, strip: bool = True) -> List[str]: If the argument is not present, returns an empty list. - The returned values are always unicode. - .. versionadded:: 3.2 """ return self._get_arguments(name, self.request.query_arguments, strip) @@ -811,7 +809,7 @@ def redirect(self, url: str, permanent: bool = False, status: int = None) -> Non def write(self, chunk: Union[str, bytes, dict]) -> None: """Writes the given chunk to the output buffer. - To write the output to the network, use the flush() method below. + To write the output to the network, use the `flush()` method below. If the given chunk is a dictionary, we write it as JSON and set the Content-Type of the response to be ``application/json``. @@ -1499,17 +1497,16 @@ def check_xsrf_cookie(self) -> None: See http://en.wikipedia.org/wiki/Cross-site_request_forgery - Prior to release 1.1.1, this check was ignored if the HTTP header - ``X-Requested-With: XMLHTTPRequest`` was present. This exception - has been shown to be insecure and has been removed. For more - information please see - http://www.djangoproject.com/weblog/2011/feb/08/security/ - http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails - .. versionchanged:: 3.2.2 Added support for cookie version 2. Both versions 1 and 2 are supported. """ + # Prior to release 1.1.1, this check was ignored if the HTTP header + # ``X-Requested-With: XMLHTTPRequest`` was present. This exception + # has been shown to be insecure and has been removed. For more + # information please see + # http://www.djangoproject.com/weblog/2011/feb/08/security/ + # http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails token = ( self.get_argument("_xsrf", None) or self.request.headers.get("X-Xsrftoken") @@ -1720,6 +1717,8 @@ def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: """Implement this method to handle streamed request data. Requires the `.stream_request_body` decorator. + + May be a coroutine for flow control. """ raise NotImplementedError() @@ -2015,7 +2014,7 @@ class Application(ReversibleRouter): Applications that do not use TLS may be vulnerable to :ref:`DNS rebinding ` attacks. This attack is especially - relevant to applications that only listen on ``127.0.0.1` or + relevant to applications that only listen on ``127.0.0.1`` or other private networks. Appropriate host patterns must be used (instead of the default of ``r'.*'``) to prevent this risk. The ``default_host`` argument must not be used in applications that diff --git a/tornado/websocket.py b/tornado/websocket.py index c77b1999ba..50665c694d 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -1496,7 +1496,7 @@ def write_message( def read_message( self, callback: Callable[["Future[Union[None, str, bytes]]"], None] = None - ) -> "Future[Union[None, str, bytes]]": + ) -> Awaitable[Union[None, str, bytes]]: """Reads a message from the WebSocket server. If on_message_callback was specified at WebSocket @@ -1508,15 +1508,17 @@ def read_message( ready. """ - future = self.read_queue.get() + awaitable = self.read_queue.get() if callback is not None: - self.io_loop.add_future(future, callback) - return future + self.io_loop.add_future(asyncio.ensure_future(awaitable), callback) + return awaitable - def on_message(self, message: Union[str, bytes]) -> Optional["Future[None]"]: + def on_message(self, message: Union[str, bytes]) -> Optional[Awaitable[None]]: return self._on_message(message) - def _on_message(self, message: Union[None, str, bytes]) -> Optional["Future[None]"]: + def _on_message( + self, message: Union[None, str, bytes] + ) -> Optional[Awaitable[None]]: if self._on_message_callback: self._on_message_callback(message) return None @@ -1580,7 +1582,7 @@ def websocket_connect( ping_timeout: float = None, max_message_size: int = _default_max_message_size, subprotocols: List[str] = None, -) -> "Future[WebSocketClientConnection]": +) -> "Awaitable[WebSocketClientConnection]": """Client-side websocket support. Takes a url and returns a Future whose result is a