From ce51302069bc87accf3d028dcbc4bf96b7300b1d Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Mon, 6 Jun 2022 23:25:11 -0400 Subject: [PATCH] Add aiohttp server and client implementation, add aiohttp server example, add aiohttp handler tests, migrate tornado tests to full async/await --- docs/md/python.md | 26 +- examples/python-aiohttp/README.md | 7 + examples/python-aiohttp/package.json | 24 + examples/python-aiohttp/server.py | 63 +++ examples/python-starlette/README.md | 2 +- examples/python-starlette/package.json | 14 +- examples/python-starlette/server.py | 10 +- .../perspective/bench/tornado/async_server.py | 2 +- python/perspective/bench/tornado/bench.py | 2 +- python/perspective/docs/perspective.core.rst | 33 +- .../perspective/perspective/client/aiohttp.py | 86 +++- .../perspective/client/starlette_test.py | 2 +- .../perspective/perspective/client/tornado.py | 5 +- .../perspective/client/websocket.py | 2 +- .../perspective/handlers/__init__.py | 2 +- .../perspective/handlers/aiohttp.py | 103 ++++- .../perspective/handlers/common.py | 83 ++++ .../perspective/handlers/starlette.py | 80 ++-- .../perspective/handlers/tornado.py | 6 +- .../perspective/perspective/tests/__init__.py | 2 +- .../perspective/tests/client/__init__.py | 7 - .../perspective/tests/client_mode/__init__.py | 2 +- .../tests/client_mode/test_client_mode.py | 371 +++++++++++----- .../perspective/perspective/tests/common.py | 38 -- .../perspective/perspective/tests/conftest.py | 40 +- .../perspective/tests/core/__init__.py | 2 +- .../perspective/tests/core/test_aggregates.py | 29 +- .../perspective/tests/core/test_async.py | 2 +- .../perspective/tests/core/test_layout.py | 15 +- .../perspective/tests/core/test_plugin.py | 6 +- .../perspective/tests/core/test_sort.py | 4 +- .../perspective/tests/core/test_threadpool.py | 12 +- .../perspective/tests/handlers/__init__.py | 7 + .../tests/handlers/test_aiohttp_handler.py | 414 +++++++++--------- .../handlers/test_aiohttp_handler_chunked.py | 126 ++++++ .../tests/handlers/test_aiohttp_lock.py | 75 ++++ .../tests/handlers/test_starlette_handler.py | 15 +- .../test_starlette_handler_chunked.py | 123 ++++++ .../tests/handlers/test_starlette_lock.py | 91 ++++ .../tests/handlers/test_tornado_handler.py | 19 +- .../handlers/test_tornado_handler_chunked.py | 54 +-- .../tests/handlers/test_tornado_lock.py | 97 ++++ .../perspective/tests/manager/__init__.py | 2 +- .../perspective/tests/table/__init__.py | 2 +- .../tests/table/test_table_pandas.py | 12 +- .../perspective/tests/viewer/__init__.py | 2 +- .../perspective/tests/widget/__init__.py | 2 +- .../tests/widget/test_widget_pandas.py | 61 ++- python/perspective/setup.cfg | 5 + python/perspective/setup.py | 12 +- 50 files changed, 1613 insertions(+), 588 deletions(-) create mode 100644 examples/python-aiohttp/README.md create mode 100644 examples/python-aiohttp/package.json create mode 100644 examples/python-aiohttp/server.py create mode 100644 python/perspective/perspective/handlers/common.py delete mode 100644 python/perspective/perspective/tests/client/__init__.py delete mode 100644 python/perspective/perspective/tests/common.py create mode 100644 python/perspective/perspective/tests/handlers/test_aiohttp_handler_chunked.py create mode 100644 python/perspective/perspective/tests/handlers/test_aiohttp_lock.py create mode 100644 python/perspective/perspective/tests/handlers/test_starlette_handler_chunked.py create mode 100644 python/perspective/perspective/tests/handlers/test_starlette_lock.py create mode 100644 python/perspective/perspective/tests/handlers/test_tornado_lock.py diff --git a/docs/md/python.md b/docs/md/python.md index ed142e8277..55e99cdea3 100644 --- a/docs/md/python.md +++ b/docs/md/python.md @@ -12,9 +12,13 @@ well as Python-specific data loading support for [NumPy](https://numpy.org/), Additionally, `perspective-python` provides a session manager suitable for integration into server systems such as -[Tornado websockets](https://www.tornadoweb.org/en/stable/websocket.html), which -allows fully _virtual_ Perspective tables to be interacted with by multiple -`` in a web browser. +[Tornado websockets](https://www.tornadoweb.org/en/stable/websocket.html), +[AIOHTTP](https://docs.aiohttp.org/en/stable/web_quickstart.html#websockets), +or [Starlette](https://www.starlette.io/websockets/)/[FastAPI](https://fastapi.tiangolo.com/advanced/websockets/), +which allows fully _virtual_ Perspective tables to be interacted with by multiple +`` in a web browser. You can also interact with a Perspective +table from python clients, and to that end client libraries are implemented for +both Tornado and AIOHTTP. As `` will only consume the data necessary to render the current screen, this runtime mode allows _ludicrously-sized_ datasets with @@ -31,9 +35,13 @@ The `perspective` module exports several tools: - `Table`, the table constructor for Perspective, which implements the `table` and `view` API in the same manner as the JavaScript library. - `PerspectiveWidget` the JupyterLab widget for interactive visualization. -- `PerspectiveTornadoHandler`, an integration with - [Tornado](https://www.tornadoweb.org/) that interfaces seamlessly with +- Perspective webserver handlers that interfaces seamlessly with `` in JavaScript. + - `PerspectiveTornadoHandler` for [Tornado](https://www.tornadoweb.org/) + - `PerspectiveStarletteHandler` for [Starlette](https://www.starlette.io/) and [FastAPI](https://fastapi.tiangolo.com) + - `PerspectiveAIOHTTPHandler` for [AIOHTTP](https://docs.aiohttp.org), + - `tornado_websocket`, a Tornado-based websocket client + - `aiohttp_websocket` an AIOHTTP-based websocket client - `PerspectiveManager` the session manager for a shared server deployment of `perspective-python`. @@ -342,7 +350,7 @@ Using Tornado and as well as `Perspective`'s JavaScript library, we can set up "distributed" Perspective instances that allows multiple browser `perspective-viewer` clients to read from a common `perspective-python` server, as in the -[Tornado Example Project](https://github.com/finos/perspective/tree/master/examples/tornado-python). +[Tornado Example Project](https://github.com/finos/perspective/tree/master/examples/python-tornado). This architecture works by maintaining two `Tables`—one on the server, and one on the client that mirrors the server's `Table` automatically using `on_update`. @@ -409,7 +417,11 @@ _*index.html*_ For a more complex example that offers distributed editing of the server dataset, see -[client_server_editing.html](https://github.com/finos/perspective/blob/master/examples/tornado-python/client_server_editing.html). +[client_server_editing.html](https://github.com/finos/perspective/blob/master/examples/python-tornado/client_server_editing.html). + +We also provide examples for Starlette/FastAPI and AIOHTTP: +- [Starlette Example Project](https://github.com/finos/perspective/tree/master/examples/python-starlette). +- [AIOHTTP Example Project](https://github.com/finos/perspective/tree/master/examples/python-aiohttp). ### Server-only Mode diff --git a/examples/python-aiohttp/README.md b/examples/python-aiohttp/README.md new file mode 100644 index 0000000000..6adf32ed92 --- /dev/null +++ b/examples/python-aiohttp/README.md @@ -0,0 +1,7 @@ +# python-aiohttp + +This example contains a simple `perspective-python` folder that uses AIOHTTP to serve a static dataset to the user through various [data bindings](https://perspective.finos.org/docs/md/server.html): + +- `index.html`: a [client/server replicated](https://perspective.finos.org/docs/md/server.html#clientserver-replicated) setup that synchronizes the client and server data using Apache Arrow. +- `server_mode.html`: a [server-only](https://perspective.finos.org/docs/md/server.html#server-only) setup that reads data and performs operations directly on the server using commands sent through the Websocket. +- `client_server_editing`: a client-server replicated setup that also enables editing, where edits from multiple clients are applied properly to the server, and then synchronized back to the clients. diff --git a/examples/python-aiohttp/package.json b/examples/python-aiohttp/package.json new file mode 100644 index 0000000000..3698ec5454 --- /dev/null +++ b/examples/python-aiohttp/package.json @@ -0,0 +1,24 @@ +{ + "name": "python-aiohttp", + "private": true, + "version": "1.4.0", + "description": "An example of editing a `perspective-python` server from the browser.", + "scripts": { + "start": "PYTHONPATH=../../python/perspective python3 server.py" + }, + "keywords": [], + "license": "Apache-2.0", + "dependencies": { + "@finos/perspective": "^1.4.0", + "@finos/perspective-viewer": "^1.4.0", + "@finos/perspective-viewer-d3fc": "^1.4.0", + "@finos/perspective-viewer-datagrid": "^1.4.0", + "@finos/perspective-workspace": "^1.4.0", + "superstore-arrow": "^1.0.0" + }, + "devDependencies": { + "@finos/perspective-webpack-plugin": "^1.4.0", + "npm-run-all": "^4.1.3", + "rimraf": "^2.5.2" + } +} diff --git a/examples/python-aiohttp/server.py b/examples/python-aiohttp/server.py new file mode 100644 index 0000000000..3ddfa4f6a9 --- /dev/null +++ b/examples/python-aiohttp/server.py @@ -0,0 +1,63 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# +import asyncio +import os +import os.path +import logging +import threading + +from aiohttp import web + +from perspective import Table, PerspectiveManager, PerspectiveAIOHTTPHandler + + +here = os.path.abspath(os.path.dirname(__file__)) +file_path = os.path.join( + here, "..", "..", "node_modules", "superstore-arrow", "superstore.arrow" +) + + +def perspective_thread(manager): + """Perspective application thread starts its own event loop, and + adds the table with the name "data_source_one", which will be used + in the front-end.""" + psp_loop = asyncio.new_event_loop() + manager.set_loop_callback(psp_loop.call_soon_threadsafe) + with open(file_path, mode="rb") as file: + table = Table(file.read(), index="Row ID") + manager.host_table("data_source_one", table) + psp_loop.run_forever() + + +def make_app(): + manager = PerspectiveManager() + + thread = threading.Thread(target=perspective_thread, args=(manager,)) + thread.daemon = True + thread.start() + + async def websocket_handler(request): + handler = PerspectiveAIOHTTPHandler(manager=manager, request=request) + await handler.run() + + app = web.Application() + app.router.add_get("/websocket", websocket_handler) + app.router.add_static( + "/node_modules/@finos", "../../node_modules/@finos", follow_symlinks=True + ) + app.router.add_static( + "/node_modules", "../../node_modules/@finos", follow_symlinks=True + ) + app.router.add_static("/", "../python-tornado", show_index=True) + return app + + +if __name__ == "__main__": + app = make_app() + logging.critical("Listening on http://localhost:8080") + web.run_app(app, host="0.0.0.0", port=8080) diff --git a/examples/python-starlette/README.md b/examples/python-starlette/README.md index e1cb5a9493..0ac9134ece 100644 --- a/examples/python-starlette/README.md +++ b/examples/python-starlette/README.md @@ -1,4 +1,4 @@ -# starlette-python +# python-starlette This example contains a simple `perspective-python` folder that uses Starlette/FastAPI to serve a static dataset to the user through various [data bindings](https://perspective.finos.org/docs/md/server.html): diff --git a/examples/python-starlette/package.json b/examples/python-starlette/package.json index 7d1dc5e151..ba32e22895 100644 --- a/examples/python-starlette/package.json +++ b/examples/python-starlette/package.json @@ -1,7 +1,7 @@ { "name": "python-starlette", "private": true, - "version": "1.3.13", + "version": "1.4.0", "description": "An example of editing a `perspective-python` server from the browser.", "scripts": { "start": "PYTHONPATH=../../python/perspective python3 server.py" @@ -9,15 +9,15 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.3.13", - "@finos/perspective-viewer": "^1.3.13", - "@finos/perspective-viewer-d3fc": "^1.3.13", - "@finos/perspective-viewer-datagrid": "^1.3.13", - "@finos/perspective-workspace": "^1.3.13", + "@finos/perspective": "^1.4.0", + "@finos/perspective-viewer": "^1.4.0", + "@finos/perspective-viewer-d3fc": "^1.4.0", + "@finos/perspective-viewer-datagrid": "^1.4.0", + "@finos/perspective-workspace": "^1.4.0", "superstore-arrow": "^1.0.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.3.13", + "@finos/perspective-webpack-plugin": "^1.4.0", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" } diff --git a/examples/python-starlette/server.py b/examples/python-starlette/server.py index 65276a04b6..54d19312ff 100644 --- a/examples/python-starlette/server.py +++ b/examples/python-starlette/server.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. @@ -43,6 +43,7 @@ def perspective_thread(manager): manager.host_table("data_source_one", table) psp_loop.run_forever() + def make_app(): manager = PerspectiveManager() @@ -51,14 +52,14 @@ def make_app(): thread.start() async def websocket_handler(websocket: WebSocket): - handler = PerspectiveStarletteHandler(manager, websocket) + handler = PerspectiveStarletteHandler(manager=manager, websocket=websocket) await handler.run() # static_html_files = StaticFiles(directory="../python-tornado", html=True) static_html_files = StaticFiles(directory="../python-tornado", html=True) app = FastAPI() - app.add_api_websocket_route('/websocket', websocket_handler) + app.add_api_websocket_route("/websocket", websocket_handler) app.get("/node_modules/{rest_of_path:path}")(static_nodemodules_handler) app.mount("/", static_html_files) @@ -71,7 +72,8 @@ async def websocket_handler(websocket: WebSocket): ) return app + if __name__ == "__main__": app = make_app() logging.critical("Listening on http://localhost:8080") - uvicorn.run(app, host='0.0.0.0', port=8080) + uvicorn.run(app, host="0.0.0.0", port=8080) diff --git a/python/perspective/bench/tornado/async_server.py b/python/perspective/bench/tornado/async_server.py index e9164715d5..7ec506c175 100644 --- a/python/perspective/bench/tornado/async_server.py +++ b/python/perspective/bench/tornado/async_server.py @@ -77,7 +77,7 @@ def make_app(manager): [ ( r"/", - perspective.tornado_handler.PerspectiveTornadoHandler, + perspective.handlers.tornado.PerspectiveTornadoHandler, {"manager": manager}, ) ] diff --git a/python/perspective/bench/tornado/bench.py b/python/perspective/bench/tornado/bench.py index b4ced867a4..d3c325d194 100644 --- a/python/perspective/bench/tornado/bench.py +++ b/python/perspective/bench/tornado/bench.py @@ -194,7 +194,7 @@ async def client(self, client_id): ) await asyncio.sleep(delay) - psp_client = await perspective.tornado_handler.websocket(self.url) + psp_client = await perspective.client.tornado.websocket(self.url) results = [] for i in range(self.num_runs): diff --git a/python/perspective/docs/perspective.core.rst b/python/perspective/docs/perspective.core.rst index 07776a95ac..87136000c0 100644 --- a/python/perspective/docs/perspective.core.rst +++ b/python/perspective/docs/perspective.core.rst @@ -1,9 +1,9 @@ ``perspective.core`` contains modules that implements ``perspective-python`` in various environments, -most notably ``PerspectiveWidget`` and ``PerspectiveTornadoHandler``. +most notably ``PerspectiveWidget`` and the various Perspective web server handlers. Additionally, ``perspective.core`` defines several enums that provide easy access to aggregate options, different plugins, sort directions etc. -For usage of ``PerspectiveWidget`` and ``PerspectiveTornadoHandler``, see the User Guide in the sidebar. +For usage of ``PerspectiveWidget`` and the Perspective web server handlers, see the User Guide in the sidebar. .. automodule:: perspective.core :members: @@ -24,16 +24,37 @@ PerspectiveWidget :show-inheritance: :exclude-members: random -PerspectiveTornadoHandler -========================= +Perspective Webserver Handlers +================================= -``PerspectiveTornadoHandler`` is a ready-made Perspective server that interfaces seamlessly with +Perspective provides several ready-made integrations with webserver libraries that interfaces seamlessly with ``@finos/perspective-viewer`` in Javascript. -.. automodule:: perspective.tornado_handler.tornado_handler +.. automodule:: perspective.handlers.tornado :members: :show-inheritance: +.. automodule:: perspective.handlers.starlette + :members: + :show-inheritance: + +.. automodule:: perspective.handlers.aiohttp + :members: + :show-inheritance: + +Perspective Websocket Clients +============================== +Perspective also provides several client interfaces to integrate with the above Perspective webserver handlers. + +.. automodule:: perspective.client.tornado + :members: + :show-inheritance: + +.. automodule:: perspective.client.aiohttp + :members: + :show-inheritance: + + PerspectiveManager ================== diff --git a/python/perspective/perspective/client/aiohttp.py b/python/perspective/perspective/client/aiohttp.py index b4395b8093..4d2424dc69 100644 --- a/python/perspective/perspective/client/aiohttp.py +++ b/python/perspective/perspective/client/aiohttp.py @@ -1,7 +1,91 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # + +import aiohttp +import asyncio + +from .websocket import ( + PerspectiveWebsocketClient, + PerspectiveWebsocketConnection, + Periodic, +) + + +class AIOHTTPPeriodic(Periodic): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._running = True + + async def _run(self): + while self._running: + await self._callback() + await asyncio.sleep(self._interval) + + async def start(self): + return asyncio.create_task(self._run()) + + async def stop(self): + self._running = False + + +class PerspectiveAIOHTTPWebsocketConnection(PerspectiveWebsocketConnection): + def __init__(self, session=None): + self._ws = None + self._session = session or aiohttp.ClientSession() + self._run = True + + async def _receive_messages(self): + async for msg in self._ws: + if msg.type == aiohttp.WSMsgType.TEXT: + self._on_message(msg.data) + elif msg.type == aiohttp.WSMsgType.BINARY: + self._on_message(msg.data) + elif msg.type == aiohttp.WSMsgType.CLOSE: + return + + async def connect(self, url, on_message, max_message_size) -> None: + self._ws_cm = self._session.ws_connect(url) + self._ws = await self._ws_cm.__aenter__() + self._on_message = on_message + self._task = asyncio.create_task(self._receive_messages()) + + def periodic(self, callback, interval) -> Periodic: + return AIOHTTPPeriodic(callback=callback, interval=interval) + + async def write(self, message, binary=False): + if binary: + return await self._ws.send_bytes(message) + else: + return await self._ws.send_str(message) + + async def close(self): + try: + self._task.cancel() + await self._task + except asyncio.CancelledError: + ... + await self._ws.close() + + +class PerspectiveAIOHTTPClient(PerspectiveWebsocketClient): + def __init__(self, session=None): + """Create a `PerspectiveAIOHTTPClient` that interfaces with a Perspective server over a Websocket""" + super(PerspectiveAIOHTTPClient, self).__init__( + PerspectiveAIOHTTPWebsocketConnection(session=session) + ) + + +async def websocket(url, session=None): + """Create a new websocket client at the given `url`. + + Args: + session (:obj:`aiohttp.ClientSession`): An optional aiohtttp session + """ + client = PerspectiveAIOHTTPClient(session=session) + await client.connect(url) + return client diff --git a/python/perspective/perspective/client/starlette_test.py b/python/perspective/perspective/client/starlette_test.py index 900b0736f2..c11506f4d5 100644 --- a/python/perspective/perspective/client/starlette_test.py +++ b/python/perspective/perspective/client/starlette_test.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/client/tornado.py b/python/perspective/perspective/client/tornado.py index 63c43b831a..51003b219b 100644 --- a/python/perspective/perspective/client/tornado.py +++ b/python/perspective/perspective/client/tornado.py @@ -45,9 +45,8 @@ async def connect(self, url, on_message, max_message_size) -> None: def periodic(self, callback, interval) -> Periodic: return TornadoPeriodic(callback=callback, interval=interval) - @gen.coroutine - def write(self, message, binary=False): - yield self._ws.write_message(message, binary=binary) + async def write(self, message, binary=False): + return await self._ws.write_message(message, binary=binary) async def close(self): self._ws.close() diff --git a/python/perspective/perspective/client/websocket.py b/python/perspective/perspective/client/websocket.py index 2917e82251..4cbd293038 100644 --- a/python/perspective/perspective/client/websocket.py +++ b/python/perspective/perspective/client/websocket.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/handlers/__init__.py b/python/perspective/perspective/handlers/__init__.py index 4063458090..c03a29be28 100644 --- a/python/perspective/perspective/handlers/__init__.py +++ b/python/perspective/perspective/handlers/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/handlers/aiohttp.py b/python/perspective/perspective/handlers/aiohttp.py index 5a28ff6c9d..d05ff328c6 100644 --- a/python/perspective/perspective/handlers/aiohttp.py +++ b/python/perspective/perspective/handlers/aiohttp.py @@ -1,11 +1,108 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # +import asyncio +from aiohttp import web, WSMsgType -class PerspectiveAIOHTTPHandler(object): - ... +from .common import PerspectiveHandlerBase + + +class PerspectiveAIOHTTPHandler(PerspectiveHandlerBase): + """PerspectiveAIOHTTPHandler is a drop-in implementation of Perspective. + + Use it inside AIOHTTP routing to create a server-side Perspective that is + ready to receive websocket messages from the front-end `perspective-viewer`. + + The Perspective client and server will automatically keep the Websocket + alive without timing out. + + Examples: + """ + + def __init__(self, *args, **kwargs): + """Create a new instance of the PerspectiveAIOHTTPHandler with the + given Manager instance. + """ + super().__init__(*args, **kwargs) + self._request = kwargs.get("request") + + async def run(self) -> None: + try: + self._ws = web.WebSocketResponse() + await self._ws.prepare(self._request) + + async for msg in self._ws: + if msg.type == WSMsgType.TEXT: + await self.on_message(msg.data) + if msg.type == WSMsgType.BINARY: + await self.on_message(msg.data) + finally: + self.on_close() + + async def on_message(self, message): + """When the websocket receives a message, send it to the :obj:`process` + method of the `PerspectiveManager` with a reference to the :obj:`post` + callback. + """ + if message == "ping": + # Respond to ping heartbeats from the Websocket client. + await self.write_message("pong") + return + + def runner(*args, **kwargs): + return asyncio.run_coroutine_threadsafe( + self.post(*args, **kwargs), asyncio.get_event_loop() + ) + + self._session.process(message, runner) + + async def post(self, message, binary=False): + """When `post` is called by `PerspectiveManager`, serialize the data to + JSON and send it to the client. + + TODO: not clear if WS needs to be locked in aiohttp like tornado, hopefully not? + + Args: + message (:obj:`str`): a JSON-serialized string containing a message to the + front-end `perspective-viewer`. + """ + # Only send message in chunks if it passes the threshold set by the + # `PerspectiveManager`. + chunked = len(message) > self._chunk_size + + async with self._stream_lock: + if binary and chunked: + start = 0 + + while start < len(message): + end = start + self._chunk_size + if end >= len(message): + end = len(message) + + asyncio.ensure_future( + self.write_message(message[start:end], binary=True) + ) + start = end + + # Allow the loop to process heartbeats so that client sockets don't + # get closed in the middle of sending a chunk. + await asyncio.sleep(self._chunk_sleep) + else: + await self.write_message(message, binary) + + def on_close(self): + """Remove the views associated with the client when the websocket + closes. + """ + self._session.close() + + async def write_message(self, message: str, binary: bool = False) -> None: + if binary: + await self._ws.send_bytes(message) + else: + await self._ws.send_str(message) diff --git a/python/perspective/perspective/handlers/common.py b/python/perspective/perspective/handlers/common.py new file mode 100644 index 0000000000..bcabe71b2b --- /dev/null +++ b/python/perspective/perspective/handlers/common.py @@ -0,0 +1,83 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# + +import asyncio +from abc import ABC, abstractmethod + +from ..core.exception import PerspectiveError + + +class PerspectiveHandlerBase(ABC): + def __init__(self, *args, **kwargs): + """Create a new instance of the PerspectiveHandlerBase with the + given Manager instance. + + Keyword Args: + manager (:obj:`PerspectiveManager`): A `PerspectiveManager` instance. + Must be provided on initialization. + check_origin (:obj`bool`): If True, all requests will be accepted + regardless of origin. Defaults to False. + chunk_size (:obj:`int`): Binary messages will not exceed this length + (in bytes); payloads above this limit will be chunked + across multiple messages. Defaults to `16_777_216` (16MB), and + be disabled entirely with `None`. + """ + self._manager = kwargs.pop("manager", None) + self._check_origin = kwargs.pop("check_origin", False) + + # uvicorn (used by aiohttp / starlette) defaults to 16MB max payload + # https://github.com/encode/uvicorn/pull/995/files + # https://github.com/euri10/uvicorn/blob/8a9369439c15a657a7726eb291858f72d3199c60/tests/protocols/test_websocket.py#L463 + self._chunk_size = kwargs.pop("chunk_size", 16_777_216) + + # https://www.tornadoweb.org/en/stable/gen.html#tornado.gen.moment + self._chunk_sleep = kwargs.pop("_chunk_sleep", 0) + + self._session = self._manager.new_session() + + if self._manager is None: + raise PerspectiveError( + "A `PerspectiveManager` instance must be provided to the handler!" + ) + + self._stream_lock = asyncio.Lock() + + def check_origin(self, origin): + """Returns whether the handler allows requests from origins outside + of the host URL. + + Args: + origin (:obj"`bool`): a boolean that indicates whether requests + outside of the host URL should be accepted. If :obj:`True`, request + URLs will not be validated and all requests will be allowed. + Defaults to :obj:`False`. + """ + return self._check_origin + + @abstractmethod + def on_message(self, message): + """When the websocket receives a message, send it to the :obj:`process` + method of the `PerspectiveManager` with a reference to the :obj:`post` + callback. + """ + + @abstractmethod + def post(self, message, binary=False): + """When `post` is called by `PerspectiveManager`, serialize the data to + JSON and send it to the client. + + Args: + message (:obj:`str`): a JSON-serialized string containing a message to the + front-end `perspective-viewer`. + """ + + @abstractmethod + def on_close(self): + """Remove the views associated with the client when the websocket + closes. + """ diff --git a/python/perspective/perspective/handlers/starlette.py b/python/perspective/perspective/handlers/starlette.py index 7b6c68c3dd..715a700e8d 100644 --- a/python/perspective/perspective/handlers/starlette.py +++ b/python/perspective/perspective/handlers/starlette.py @@ -7,11 +7,11 @@ # import asyncio -from starlette.websockets import WebSocket -from ..core.exception import PerspectiveError +from .common import PerspectiveHandlerBase -class PerspectiveStarletteHandler(object): + +class PerspectiveStarletteHandler(PerspectiveHandlerBase): """PerspectiveStarletteHandler is a drop-in implementation of Perspective. The Perspective client and server will automatically keep the Websocket @@ -28,15 +28,8 @@ class PerspectiveStarletteHandler(object): ... app.add_api_websocket_route('/websocket', endpoint) """ - def __init__( - self, - manager, - websocket: WebSocket, - check_origin=False, - chunk_size=25165824, - chunk_sleep=0, - ): - """Create a new instance of the PerspectiveTornadoHandler with the + def __init__(self, *args, **kwargs): + """Create a new instance of the PerspectiveStarletteHandler with the given Manager instance. Keyword Args: @@ -46,24 +39,11 @@ def __init__( regardless of origin. Defaults to False. chunk_size (:obj:`int`): Binary messages will not exceed this length (in bytes); payloads above this limit will be chunked - across multiple messages. Defaults to `25165824` (24MB), and + across multiple messages. Defaults to `16_777_216` (16MB), and be disabled entirely with `None`. """ - self._manager = manager - self._check_origin = check_origin - self._chunk_size = chunk_size - self._session = self._manager.new_session() - self._websocket = websocket - self._chunk_sleep = chunk_sleep - - if self._manager is None: - raise PerspectiveError( - "A `PerspectiveManager` instance must be provided to the handler!" - ) - - def check_origin(self, origin): - """TODO""" - return self._check_origin + super().__init__(*args, **kwargs) + self._websocket = kwargs["websocket"] async def run(self) -> None: try: @@ -109,15 +89,25 @@ async def post(self, message, binary=False): # `PerspectiveManager`. chunked = len(message) > self._chunk_size - if binary and chunked: - await self._post_chunked( - message, - 0, - self._chunk_size, - len(message), - ) - else: - await self.write_message(message, binary) + async with self._stream_lock: + if binary and chunked: + start = 0 + + while start < len(message): + end = start + self._chunk_size + if end >= len(message): + end = len(message) + + asyncio.ensure_future( + self.write_message(message[start:end], binary=True) + ) + start = end + + # Allow the loop to process heartbeats so that client sockets don't + # get closed in the middle of sending a chunk. + await asyncio.sleep(self._chunk_sleep) + else: + await self.write_message(message, binary) def on_close(self): """Remove the views associated with the client when the websocket @@ -125,22 +115,6 @@ def on_close(self): """ self._session.close() - async def _post_chunked(self, message, start, end, message_length): - """Send a binary message in chunks on the websocket.""" - if start < message_length: - end = start + self._chunk_size - - if end >= message_length: - end = message_length - - await self.write_message(message[start:end], binary=True) - start = end - - # Allow the loop to process heartbeats so that client sockets don't - # get closed in the middle of sending a chunk. - await asyncio.sleep(self._chunk_sleep) - await self._post_chunked(message, start, end, message_length) - async def write_message(self, message: str, binary: bool = False) -> None: if binary: await self._websocket.send_bytes(message) diff --git a/python/perspective/perspective/handlers/tornado.py b/python/perspective/perspective/handlers/tornado.py index 0a11245b85..190d0e657d 100644 --- a/python/perspective/perspective/handlers/tornado.py +++ b/python/perspective/perspective/handlers/tornado.py @@ -13,7 +13,7 @@ from tornado.gen import coroutine from tornado.ioloop import IOLoop -from ..core.exception import PerspectiveError +from perspective import PerspectiveError class PerspectiveTornadoHandler(tornado.websocket.WebSocketHandler): @@ -42,7 +42,7 @@ class PerspectiveTornadoHandler(tornado.websocket.WebSocketHandler): """ def __init__(self, *args, **kwargs): - """Create a new instance of the PerspectiveTornadoHandler with the + """Create a new instance of the PerspectiveHandlerBase with the given Manager instance. Keyword Args: @@ -69,7 +69,7 @@ def __init__(self, *args, **kwargs): "A `PerspectiveManager` instance must be provided to the tornado handler!" ) - super(PerspectiveTornadoHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def check_origin(self, origin): """Returns whether the handler allows requests from origins outside diff --git a/python/perspective/perspective/tests/__init__.py b/python/perspective/perspective/tests/__init__.py index b4395b8093..42db35c427 100644 --- a/python/perspective/perspective/tests/__init__.py +++ b/python/perspective/perspective/tests/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/tests/client/__init__.py b/python/perspective/perspective/tests/client/__init__.py deleted file mode 100644 index b4395b8093..0000000000 --- a/python/perspective/perspective/tests/client/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -################################################################################ -# -# Copyright (c) 2022, the Perspective Authors. -# -# This file is part of the Perspective library, distributed under the terms of -# the Apache License 2.0. The full license can be found in the LICENSE file. -# diff --git a/python/perspective/perspective/tests/client_mode/__init__.py b/python/perspective/perspective/tests/client_mode/__init__.py index b4395b8093..42db35c427 100644 --- a/python/perspective/perspective/tests/client_mode/__init__.py +++ b/python/perspective/perspective/tests/client_mode/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/tests/client_mode/test_client_mode.py b/python/perspective/perspective/tests/client_mode/test_client_mode.py index 5ffbd91e25..8e99975018 100644 --- a/python/perspective/perspective/tests/client_mode/test_client_mode.py +++ b/python/perspective/perspective/tests/client_mode/test_client_mode.py @@ -7,19 +7,22 @@ # import os +import pytest +import sys +import numpy as np +import pandas as pd + from datetime import date, datetime from functools import partial from types import MethodType -import numpy as np -import pandas as pd -if os.name == 'nt': - BINDING = 'libbinding.pyd' - PSP = 'libpsp.dll' +if os.name == "nt": + BINDING = "libbinding.pyd" + PSP = "libpsp.dll" else: - BINDING = 'libbinding.so' - PSP = 'libpsp.so' + BINDING = "libbinding.so" + PSP = "libpsp.so" # rename libbinding.so and libpsp.so temporarily to ensure that client mode # works automatically when the C++ build fails. @@ -31,11 +34,26 @@ def mock_post(self, msg, msg_id=None, assert_msg=None): - '''Mock the widget's `post()` method so we can introspect the contents.''' + """Mock the widget's `post()` method so we can introspect the contents.""" assert msg == assert_msg -def setup_module(): +def unload(): + to_pop = [] + for mod in sys.modules: + if mod.startswith("perspective"): + to_pop.append(mod) + for mod in to_pop: + sys.modules.pop(mod) + + +@pytest.fixture(scope="class") +def rename_libraries(): + print("SETTING UP") + # unload perspective from sys modules + unload() + + # rename the binding so it doesnt import os.rename(binding, new_binding) os.rename(psp, new_psp) assert os.path.exists(new_binding) @@ -43,8 +61,13 @@ def setup_module(): assert not os.path.exists(binding) assert not os.path.exists(psp) + # import + import perspective + + # defer to test + yield -def teardown_module(): + # rename back os.rename(new_binding, binding) os.rename(new_psp, psp) assert os.path.exists(binding) @@ -52,46 +75,55 @@ def teardown_module(): assert not os.path.exists(new_binding) assert not os.path.exists(new_psp) + # unload from sys.modules + unload() + + # import again + import perspective -class TestClient(object): - def test_widget_client(self): +class TestClient(object): + def test_widget_client(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": [i for i in range(50)]} widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == data - def test_widget_client_np(self): + def test_widget_client_np(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": np.arange(0, 50)} widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False - assert widget._data == { - "a": [i for i in range(50)] - } + assert widget._data == {"a": [i for i in range(50)]} - def test_widget_client_df(self): + def test_widget_client_df(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = pd.DataFrame({ - "a": np.arange(10), - "b": [True for i in range(10)], - "c": [str(i) for i in range(10)] - }) + data = pd.DataFrame( + { + "a": np.arange(10), + "b": [True for i in range(10)], + "c": [str(i) for i in range(10)], + } + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "index": [i for i in range(10)], "a": [i for i in range(10)], "b": [True for i in range(10)], - "c": [str(i) for i in range(10)] + "c": [str(i) for i in range(10)], } - def test_widget_client_date(self): + def test_widget_client_date(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": [date(2020, i, 1) for i in range(1, 13)]} widget = perspective.PerspectiveWidget(data) @@ -99,58 +131,61 @@ def test_widget_client_date(self): # `data` is mutated at this point, so check against the expected # formatting just to make sure. - assert widget._data == { - "a": ["2020-{:02d}-01".format(i) for i in range(1, 13)] - } + assert widget._data == {"a": ["2020-{:02d}-01".format(i) for i in range(1, 13)]} - def test_widget_client_np_date(self): + def test_widget_client_np_date(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = {"a": np.array([date(2020, i, 1) for i in range(1, 13)], dtype="datetime64[D]")} + data = { + "a": np.array( + [date(2020, i, 1) for i in range(1, 13)], dtype="datetime64[D]" + ) + } widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False - assert widget._data == { - "a": ["2020-{:02d}-01".format(i) for i in range(1, 13)] - } + assert widget._data == {"a": ["2020-{:02d}-01".format(i) for i in range(1, 13)]} - def test_widget_client_np_date_object(self): + def test_widget_client_np_date_object(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": np.array([date(2020, i, 1) for i in range(1, 13)], dtype="object")} widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False - assert widget._data == { - "a": ["2020-{:02d}-01".format(i) for i in range(1, 13)] - } + assert widget._data == {"a": ["2020-{:02d}-01".format(i) for i in range(1, 13)]} - def test_widget_client_df_date(self): + def test_widget_client_df_date(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = pd.DataFrame({ - "a": [date(2020, i, 1) for i in range(1, 13)] - }, dtype="datetime64[ns]") + data = pd.DataFrame( + {"a": [date(2020, i, 1) for i in range(1, 13)]}, dtype="datetime64[ns]" + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "index": [i for i in range(12)], - "a": ["2020-{:02d}-01 00:00:00".format(i) for i in range(1, 13)] + "a": ["2020-{:02d}-01 00:00:00".format(i) for i in range(1, 13)], } - def test_widget_client_df_date_object(self): + def test_widget_client_df_date_object(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = pd.DataFrame({ - "a": [date(2020, i, 1) for i in range(1, 13)] - }, dtype="object") + data = pd.DataFrame( + {"a": [date(2020, i, 1) for i in range(1, 13)]}, dtype="object" + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "index": [i for i in range(12)], - "a": ["2020-{:02d}-01".format(i) for i in range(1, 13)] + "a": ["2020-{:02d}-01".format(i) for i in range(1, 13)], } - def test_widget_client_datetime(self): + def test_widget_client_datetime(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)]} widget = perspective.PerspectiveWidget(data) @@ -159,129 +194,166 @@ def test_widget_client_datetime(self): "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)] } - def test_widget_client_np_datetime(self): + def test_widget_client_np_datetime(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = {"a": np.array([datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)], dtype="datetime64")} + data = { + "a": np.array( + [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)], + dtype="datetime64", + ) + } widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)] } - def test_widget_client_np_datetime_object(self): + def test_widget_client_np_datetime_object(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = {"a": np.array([datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)], dtype="object")} + data = { + "a": np.array( + [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)], dtype="object" + ) + } widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)] } - def test_widget_client_df_datetime(self): + def test_widget_client_df_datetime(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = pd.DataFrame({ - "a": [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)] - }, dtype="datetime64[ns]") + data = pd.DataFrame( + {"a": [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)]}, + dtype="datetime64[ns]", + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "index": [i for i in range(12)], - "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)] + "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)], } - def test_widget_client_df_datetime_object(self): + def test_widget_client_df_datetime_object(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = pd.DataFrame({ - "a": [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)] - }, dtype="object") + data = pd.DataFrame( + {"a": [datetime(2020, i, 1, 12, 30, 45) for i in range(1, 13)]}, + dtype="object", + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "index": [i for i in range(12)], - "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)] + "a": ["2020-{:02d}-01 12:30:45".format(i) for i in range(1, 13)], } - def test_widget_client_np_structured_array(self): + def test_widget_client_np_structured_array(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = np.array([(1, 2), (3, 4)], dtype=[("a", "int64"), ("b", "int64")]) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False - assert widget._data == { - "a": [1, 3], - "b": [2, 4] - } + assert widget._data == {"a": [1, 3], "b": [2, 4]} - def test_widget_client_np_recarray(self): + def test_widget_client_np_recarray(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = np.array([(1, 2), (3, 4)], dtype=[("a", "int64"), ("b", "int64")]).view(np.recarray) + data = np.array([(1, 2), (3, 4)], dtype=[("a", "int64"), ("b", "int64")]).view( + np.recarray + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False - assert widget._data == { - "a": [1, 3], - "b": [2, 4] - } + assert widget._data == {"a": [1, 3], "b": [2, 4]} - def test_widget_client_np_structured_array_date(self): + def test_widget_client_np_structured_array_date(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = np.array([(date(2020, 1, 1), date(2020, 2, 1)), (date(2020, 3, 1), date(2020, 4, 1))], dtype=[("a", "datetime64[D]"), ("b", "datetime64[D]")]) + data = np.array( + [ + (date(2020, 1, 1), date(2020, 2, 1)), + (date(2020, 3, 1), date(2020, 4, 1)), + ], + dtype=[("a", "datetime64[D]"), ("b", "datetime64[D]")], + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "a": ["2020-01-01", "2020-03-01"], - "b": ["2020-02-01", "2020-04-01"] + "b": ["2020-02-01", "2020-04-01"], } - def test_widget_client_np_recarray_date(self): + def test_widget_client_np_recarray_date(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = np.array([(date(2020, 1, 1), date(2020, 2, 1)), (date(2020, 3, 1), date(2020, 4, 1))], dtype=[("a", "datetime64[D]"), ("b", "datetime64[D]")]) + data = np.array( + [ + (date(2020, 1, 1), date(2020, 2, 1)), + (date(2020, 3, 1), date(2020, 4, 1)), + ], + dtype=[("a", "datetime64[D]"), ("b", "datetime64[D]")], + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "a": ["2020-01-01", "2020-03-01"], - "b": ["2020-02-01", "2020-04-01"] + "b": ["2020-02-01", "2020-04-01"], } - def test_widget_client_np_structured_array_date_object(self): + def test_widget_client_np_structured_array_date_object(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = np.array([(date(2020, 1, 1), date(2020, 2, 1)), (date(2020, 3, 1), date(2020, 4, 1))], dtype=[("a", "object"), ("b", "object")]) + data = np.array( + [ + (date(2020, 1, 1), date(2020, 2, 1)), + (date(2020, 3, 1), date(2020, 4, 1)), + ], + dtype=[("a", "object"), ("b", "object")], + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "a": ["2020-01-01", "2020-03-01"], - "b": ["2020-02-01", "2020-04-01"] + "b": ["2020-02-01", "2020-04-01"], } - def test_widget_client_np_recarray_date_object(self): + def test_widget_client_np_recarray_date_object(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - data = np.array([(date(2020, 1, 1), date(2020, 2, 1)), (date(2020, 3, 1), date(2020, 4, 1))], dtype=[("a", "object"), ("b", "object")]) + data = np.array( + [ + (date(2020, 1, 1), date(2020, 2, 1)), + (date(2020, 3, 1), date(2020, 4, 1)), + ], + dtype=[("a", "object"), ("b", "object")], + ) widget = perspective.PerspectiveWidget(data) assert hasattr(widget, "table") is False assert widget._data == { "a": ["2020-01-01", "2020-03-01"], - "b": ["2020-02-01", "2020-04-01"] + "b": ["2020-02-01", "2020-04-01"], } - def test_widget_client_schema(self): + def test_widget_client_schema(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False - widget = perspective.PerspectiveWidget({ - "a": int, - "b": float, - "c": bool, - "d": date, - "e": datetime, - "f": str - }) + widget = perspective.PerspectiveWidget( + {"a": int, "b": float, "c": bool, "d": date, "e": datetime, "f": str} + ) assert hasattr(widget, "table") is False assert widget._data == { "a": "integer", @@ -289,62 +361,121 @@ def test_widget_client_schema(self): "c": "boolean", "d": "date", "e": "datetime", - "f": "string" + "f": "string", } - def test_widget_client_update(self): + def test_widget_client_update(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": np.arange(0, 50)} - comparison_data = { - "a": [i for i in range(50)] - } + comparison_data = {"a": [i for i in range(50)]} widget = perspective.PerspectiveWidget(data) - mocked_post = partial(mock_post, assert_msg={ - "cmd": "update", - "data": comparison_data - }) + mocked_post = partial( + mock_post, assert_msg={"cmd": "update", "data": comparison_data} + ) widget.post = MethodType(mocked_post, widget) widget.update(data) assert hasattr(widget, "table") is False - def test_widget_client_replace(self): + def test_widget_client_replace(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": np.arange(0, 50)} new_data = {"a": [1]} widget = perspective.PerspectiveWidget(data) - mocked_post = partial(mock_post, assert_msg={ - "cmd": "replace", - "data": new_data - }) + mocked_post = partial( + mock_post, assert_msg={"cmd": "replace", "data": new_data} + ) widget.post = MethodType(mocked_post, widget) widget.replace(new_data) assert widget._data is new_data - def test_widget_delete_client(self): + def test_widget_delete_client(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False data = {"a": np.arange(0, 50)} widget = perspective.PerspectiveWidget(data) - mocked_post = partial(mock_post, assert_msg={ - "cmd": "delete" - }) + mocked_post = partial(mock_post, assert_msg={"cmd": "delete"}) widget.delete() widget.post = MethodType(mocked_post, widget) - def test_widget_load_split_by_client(self): + def test_widget_load_split_by_client(self, rename_libraries): import perspective + assert perspective.is_libpsp() is False # behavior should not change for client mode - arrays = [np.array(['bar', 'bar', 'bar', 'bar', 'baz', 'baz', 'baz', 'baz', 'foo', 'foo', 'foo', 'foo', 'qux', 'qux', 'qux', 'qux']), - np.array(['one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two']), - np.array(['X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y'])] + arrays = [ + np.array( + [ + "bar", + "bar", + "bar", + "bar", + "baz", + "baz", + "baz", + "baz", + "foo", + "foo", + "foo", + "foo", + "qux", + "qux", + "qux", + "qux", + ] + ), + np.array( + [ + "one", + "one", + "two", + "two", + "one", + "one", + "two", + "two", + "one", + "one", + "two", + "two", + "one", + "one", + "two", + "two", + ] + ), + np.array( + [ + "X", + "Y", + "X", + "Y", + "X", + "Y", + "X", + "Y", + "X", + "Y", + "X", + "Y", + "X", + "Y", + "X", + "Y", + ] + ), + ] tuples = list(zip(*arrays)) - index = pd.MultiIndex.from_tuples(tuples, names=['first', 'second', 'third']) - df_both = pd.DataFrame(np.random.randn(3, 16), index=['A', 'B', 'C'], columns=index) + index = pd.MultiIndex.from_tuples(tuples, names=["first", "second", "third"]) + df_both = pd.DataFrame( + np.random.randn(3, 16), index=["A", "B", "C"], columns=index + ) widget = perspective.PerspectiveWidget(df_both) assert hasattr(widget, "table") is False - assert widget.columns == ['value'] - assert widget.split_by == ['first', 'second', 'third'] - assert widget.group_by == ['index'] + assert widget.columns == ["value"] + assert widget.split_by == ["first", "second", "third"] + assert widget.group_by == ["index"] diff --git a/python/perspective/perspective/tests/common.py b/python/perspective/perspective/tests/common.py deleted file mode 100644 index 646018b972..0000000000 --- a/python/perspective/perspective/tests/common.py +++ /dev/null @@ -1,38 +0,0 @@ -import pandas as pd -from random import random, randint, choice -from faker import Faker - -fake = Faker() - - -def superstore(count=50): - data = [] - for id in range(count): - dat = {} - dat["Row ID"] = id - dat["Order ID"] = "{}-{}".format(fake.ein(), fake.zipcode()) - dat["Order Date"] = fake.date_this_year() - dat["Ship Date"] = fake.date_between_dates(dat["Order Date"]).strftime( - "%Y-%m-%d" - ) - dat["Order Date"] = dat["Order Date"].strftime("%Y-%m-%d") - dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"]) - dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"]) - dat["Customer ID"] = fake.zipcode() - dat["Segment"] = choice(["A", "B", "C", "D"]) - dat["Country"] = "US" - dat["City"] = fake.city() - dat["State"] = fake.state() - dat["Postal Code"] = fake.zipcode() - dat["Region"] = choice(["Region %d" % i for i in range(5)]) - dat["Product ID"] = fake.bban() - sector = choice(["Industrials", "Technology", "Financials"]) - industry = choice(["A", "B", "C"]) - dat["Category"] = sector - dat["Sub-Category"] = industry - dat["Sales"] = randint(1, 100) * 100 - dat["Quantity"] = randint(1, 100) * 10 - dat["Discount"] = round(random() * 100, 2) - dat["Profit"] = round(random() * 1000, 2) - data.append(dat) - return pd.DataFrame(data) diff --git a/python/perspective/perspective/tests/conftest.py b/python/perspective/perspective/tests/conftest.py index c082cec001..93f9ffaca7 100644 --- a/python/perspective/perspective/tests/conftest.py +++ b/python/perspective/perspective/tests/conftest.py @@ -6,13 +6,17 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import time from datetime import datetime import numpy as np import pandas as pd import pyarrow as pa from pytest import fixture +from random import random, randint, choice +from faker import Faker + + +fake = Faker() def _make_date_time_index(size, time_unit): @@ -202,3 +206,37 @@ def _sentinel(value): def util(): """Pass the `Util` class in to a test.""" return Util + + +@fixture +def superstore(count=100): + data = [] + for id in range(count): + dat = {} + dat["Row ID"] = id + dat["Order ID"] = "{}-{}".format(fake.ein(), fake.zipcode()) + dat["Order Date"] = fake.date_this_year() + dat["Ship Date"] = fake.date_between_dates(dat["Order Date"]).strftime( + "%Y-%m-%d" + ) + dat["Order Date"] = dat["Order Date"].strftime("%Y-%m-%d") + dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"]) + dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"]) + dat["Customer ID"] = fake.zipcode() + dat["Segment"] = choice(["A", "B", "C", "D"]) + dat["Country"] = "US" + dat["City"] = fake.city() + dat["State"] = fake.state() + dat["Postal Code"] = fake.zipcode() + dat["Region"] = choice(["Region %d" % i for i in range(5)]) + dat["Product ID"] = fake.bban() + sector = choice(["Industrials", "Technology", "Financials"]) + industry = choice(["A", "B", "C"]) + dat["Category"] = sector + dat["Sub-Category"] = industry + dat["Sales"] = randint(1, 100) * 100 + dat["Quantity"] = randint(1, 100) * 10 + dat["Discount"] = round(random() * 100, 2) + dat["Profit"] = round(random() * 1000, 2) + data.append(dat) + return pd.DataFrame(data) diff --git a/python/perspective/perspective/tests/core/__init__.py b/python/perspective/perspective/tests/core/__init__.py index b4395b8093..42db35c427 100644 --- a/python/perspective/perspective/tests/core/__init__.py +++ b/python/perspective/perspective/tests/core/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/tests/core/test_aggregates.py b/python/perspective/perspective/tests/core/test_aggregates.py index e72d658dd6..2404c1c0d6 100644 --- a/python/perspective/perspective/tests/core/test_aggregates.py +++ b/python/perspective/perspective/tests/core/test_aggregates.py @@ -7,23 +7,26 @@ # from pytest import raises -from perspective import ( - PerspectiveError, - PerspectiveViewer, - PerspectiveWidget, - Aggregate, -) +from perspective import PerspectiveError, PerspectiveViewer,\ + PerspectiveWidget, Aggregate class TestAggregates: + def test_aggregates_widget_load(self): - aggs = {"a": Aggregate.AVG, "b": Aggregate.LAST} + aggs = { + "a": Aggregate.AVG, + "b": Aggregate.LAST + } data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} widget = PerspectiveWidget(data, aggregates=aggs) assert widget.aggregates == aggs def test_aggregates_widget_load_weighted_mean(self): - aggs = {"a": Aggregate.AVG, "b": ["weighted mean", "a"]} + aggs = { + "a": Aggregate.AVG, + "b": ["weighted mean", "a"] + } data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} widget = PerspectiveWidget(data, aggregates=aggs) assert widget.aggregates == aggs @@ -31,8 +34,14 @@ def test_aggregates_widget_load_weighted_mean(self): def test_aggregates_widget_setattr(self): data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} widget = PerspectiveWidget(data) - widget.aggregates = {"a": Aggregate.ANY, "b": Aggregate.LAST} - assert widget.aggregates == {"a": "any", "b": "last"} + widget.aggregates = { + "a": Aggregate.ANY, + "b": Aggregate.LAST + } + assert widget.aggregates == { + "a": "any", + "b": "last" + } def test_aggregates_widget_load_invalid(self): data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} diff --git a/python/perspective/perspective/tests/core/test_async.py b/python/perspective/perspective/tests/core/test_async.py index 2c7313ffde..f79edb0208 100644 --- a/python/perspective/perspective/tests/core/test_async.py +++ b/python/perspective/perspective/tests/core/test_async.py @@ -74,7 +74,7 @@ def _task(): tbl.delete() def test_async_queue_process_csv(self): - """Make sure GIL release during CSV loading works""" + """Make sure GIL release during CSV loading works""" tbl = Table("x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false") manager = PerspectiveManager() manager.set_loop_callback(TestAsync.loop.add_callback) diff --git a/python/perspective/perspective/tests/core/test_layout.py b/python/perspective/perspective/tests/core/test_layout.py index d06d2e99ff..57a22b8d80 100644 --- a/python/perspective/perspective/tests/core/test_layout.py +++ b/python/perspective/perspective/tests/core/test_layout.py @@ -12,13 +12,14 @@ class TestLayout: + def test_layout_invalid_plugin(self): - with patch("IPython.display.display"): - df = pd.DataFrame([1, 2], columns=["1"]) + with patch('IPython.display.display'): + df = pd.DataFrame([1, 2], columns=['1']) PerspectiveWidget(df, plugin=Plugin.YBAR) - PerspectiveWidget(df, plugin="Y Line") + PerspectiveWidget(df, plugin='Y Line') try: - PerspectiveWidget(df, plugin="test") + PerspectiveWidget(df, plugin='test') assert False except PerspectiveError: pass @@ -30,9 +31,9 @@ def test_layout_invalid_plugin(self): pass def test_layout_invalid_columns(self): - with patch("IPython.display.display"): - df = pd.DataFrame([1, 2], columns=["1"]) - PerspectiveWidget(df, plugin=Plugin.YBAR, columns=["1"]) + with patch('IPython.display.display'): + df = pd.DataFrame([1, 2], columns=['1']) + PerspectiveWidget(df, plugin=Plugin.YBAR, columns=['1']) try: PerspectiveWidget(df, plugin=Plugin.YBAR, columns=5) assert False diff --git a/python/perspective/perspective/tests/core/test_plugin.py b/python/perspective/perspective/tests/core/test_plugin.py index 4f67f56956..d15e3ce86e 100644 --- a/python/perspective/perspective/tests/core/test_plugin.py +++ b/python/perspective/perspective/tests/core/test_plugin.py @@ -7,10 +7,12 @@ # from pytest import raises -from perspective import PerspectiveError, PerspectiveViewer, PerspectiveWidget, Plugin +from perspective import PerspectiveError, PerspectiveViewer,\ + PerspectiveWidget, Plugin class TestPlugin: + def test_plugin_widget_load_grid(self): data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} widget = PerspectiveWidget(data, plugin=Plugin.GRID) @@ -34,7 +36,7 @@ def test_plugin_widget_load_invalid(self): def test_plugin_widget_setattr_invalid(self): data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} - widget = PerspectiveWidget(data) + widget = PerspectiveWidget(data) with raises(PerspectiveError): widget.plugin = "?" diff --git a/python/perspective/perspective/tests/core/test_sort.py b/python/perspective/perspective/tests/core/test_sort.py index 6354d2a953..403e5a59f4 100644 --- a/python/perspective/perspective/tests/core/test_sort.py +++ b/python/perspective/perspective/tests/core/test_sort.py @@ -7,10 +7,12 @@ # from pytest import raises -from perspective import PerspectiveError, PerspectiveViewer, PerspectiveWidget, Sort +from perspective import PerspectiveError, PerspectiveViewer,\ + PerspectiveWidget, Sort class TestSort(object): + def test_sort_widget_load(self): data = {"a": [1, 2, 3], "b": ["a", "b", "c"]} widget = PerspectiveWidget(data, sort=[["a", Sort.DESC]]) diff --git a/python/perspective/perspective/tests/core/test_threadpool.py b/python/perspective/perspective/tests/core/test_threadpool.py index 28cbc4f195..348a78a2ad 100644 --- a/python/perspective/perspective/tests/core/test_threadpool.py +++ b/python/perspective/perspective/tests/core/test_threadpool.py @@ -8,13 +8,11 @@ from perspective import Table, set_threadpool_size - def compare_delta(received, expected): """Compare an arrow-serialized row delta by constructing a Table.""" tbl = Table(received) assert tbl.view().to_dict() == expected - class TestThreadpool(object): def test_set_threadpool_size(self): set_threadpool_size(1) @@ -23,7 +21,10 @@ def test_set_threadpool_size(self): view = tbl.view() assert view.num_rows() == 2 assert view.num_columns() == 2 - assert view.schema() == {"a": int, "b": int} + assert view.schema() == { + "a": int, + "b": int + } assert view.to_records() == data def test_set_threadpool_size_max(self): @@ -33,5 +34,8 @@ def test_set_threadpool_size_max(self): view = tbl.view() assert view.num_rows() == 2 assert view.num_columns() == 2 - assert view.schema() == {"a": int, "b": int} + assert view.schema() == { + "a": int, + "b": int + } assert view.to_records() == data diff --git a/python/perspective/perspective/tests/handlers/__init__.py b/python/perspective/perspective/tests/handlers/__init__.py index e69de29bb2..42db35c427 100644 --- a/python/perspective/perspective/tests/handlers/__init__.py +++ b/python/perspective/perspective/tests/handlers/__init__.py @@ -0,0 +1,7 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# diff --git a/python/perspective/perspective/tests/handlers/test_aiohttp_handler.py b/python/perspective/perspective/tests/handlers/test_aiohttp_handler.py index e034a9751c..433c475418 100644 --- a/python/perspective/perspective/tests/handlers/test_aiohttp_handler.py +++ b/python/perspective/perspective/tests/handlers/test_aiohttp_handler.py @@ -5,17 +5,22 @@ # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # + +# NOTE: This is essentially a clone of the tornado handler tests, +# but using async/await and starlette handler import random import pytest -import tornado -from tornado import gen +from aiohttp import web from datetime import datetime -from ...core.exception import PerspectiveError -from ...table import Table -from ...manager import PerspectiveManager -from ...handlers.aiohttp import PerspectiveAIOHTTPHandler, websocket +from perspective import ( + Table, + PerspectiveManager, + PerspectiveAIOHTTPHandler, + aiohttp_websocket as websocket, + PerspectiveError, +) data = { @@ -27,289 +32,274 @@ MANAGER = PerspectiveManager() -APPLICATION = tornado.web.Application( - [ - ( - r"/websocket", - PerspectiveTornadoHandler, - {"manager": MANAGER, "check_origin": True}, - ) - ] -) +async def websocket_handler(request): + handler = PerspectiveAIOHTTPHandler(manager=MANAGER, request=request) + await handler.run() -@pytest.fixture(scope="module") + +@pytest.fixture def app(): - return APPLICATION + app = web.Application() + app.router.add_get("/websocket", websocket_handler) + return app -class TestPerspectiveTornadoHandler(object): +class TestPerspectiveAIOHTTPHandler(object): def setup_method(self): """Flush manager state before each test method execution.""" MANAGER._tables = {} MANAGER._views = {} - @gen.coroutine - def websocket_client(self, port): + async def websocket_client(self, app, aiohttp_client): """Connect and initialize a websocket client connection to the - Perspective tornado server. + Perspective aiottp server. """ - client = yield websocket( - "ws://127.0.0.1:{0}/websocket".format(port) + client = await aiohttp_client(app) + return await websocket( + "http://{}:{}/websocket".format(client.host, client.port), client.session ) - # Compatibility with Python < 3.3 - raise gen.Return(client) - - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_init_terminate(self, app, http_client, http_port): - # """Using Tornado's websocket client, test the websocket provided by - # PerspectiveTornadoHandler. - - # All test methods must import `app`, `http_client`, and `http_port`, - # otherwise a mysterious timeout will occur.""" - # client = yield self.websocket_client(http_port) - # client.terminate() + @pytest.mark.asyncio + async def test_aiohttp_handler_init_terminate(self, app, aiohttp_client): + """Using AIOHTTP's websocket client, test the websocket provided by + PerspectiveAIOHTTPHandler""" + client = await self.websocket_client(app, aiohttp_client) + await client.terminate() - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_table_method(self, app, http_client, http_port): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_table_method(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) - # schema = yield table.schema() - # size = yield table.size() + schema = await table.schema() + size = await table.size() - # assert schema == { - # "a": "integer", - # "b": "float", - # "c": "string", - # "d": "datetime", - # } + assert schema == { + "a": "integer", + "b": "float", + "c": "string", + "d": "datetime", + } - # assert size == 10 + assert size == 10 - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_make_table(self, app, http_client, http_port): - # client = yield self.websocket_client(http_port) - # table = yield client.table(data) - # size = yield table.size() + @pytest.mark.asyncio + async def test_aiohttp_handler_make_table(self, app, aiohttp_client): + client = await self.websocket_client(app, aiohttp_client) + table = await client.table(data) + size = await table.size() - # assert size == 10 + assert size == 10 - # table.update(data) + table.update(data) - # size2 = yield table.size() - # assert size2 == 20 + size2 = await table.size() + assert size2 == 20 - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_table_update(self, app, http_client, http_port): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_table_update(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # size = yield table.size() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + size = await table.size() - # assert size == 10 + assert size == 10 - # table.update(data) + table.update(data) - # size2 = yield table.size() - # assert size2 == 20 + size2 = await table.size() + assert size2 == 20 - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_table_update_port( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_table_update_port( + self, app, aiohttp_client, sentinel + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # view = yield table.view() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() - # size = yield table.size() + size = await table.size() - # assert size == 10 + assert size == 10 - # for i in range(5): - # yield table.make_port() + for i in range(5): + await table.make_port() - # port = yield table.make_port() + port = await table.make_port() - # s = sentinel(False) + s = sentinel(False) - # def updater(port_id): - # s.set(True) - # assert port_id == port + def updater(port_id): + s.set(True) + assert port_id == port - # view.on_update(updater) + view.on_update(updater) - # table.update(data, port_id=port) + table.update(data, port_id=port) - # size2 = yield table.size() - # assert size2 == 20 - # assert s.get() is True + size2 = await table.size() + assert size2 == 20 + assert s.get() is True - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_table_update_row_delta( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_table_update_row_delta( + self, app, aiohttp_client, sentinel + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # view = yield table.view() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() - # size = yield table.size() + size = await table.size() - # assert size == 10 + assert size == 10 - # s = sentinel(False) + s = sentinel(False) - # def updater(port_id, delta): - # s.set(True) - # t2 = Table(delta) - # assert t2.view().to_dict() == data - # assert port_id == 0 + def updater(port_id, delta): + s.set(True) + t2 = Table(delta) + assert t2.view().to_dict() == data + assert port_id == 0 - # view.on_update(updater, mode="row") + view.on_update(updater, mode="row") - # table.update(data) + table.update(data) - # size2 = yield table.size() - # assert size2 == 20 - # assert s.get() is True + size2 = await table.size() + assert size2 == 20 + assert s.get() is True - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_table_update_row_delta_port( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_table_update_row_delta_port( + self, app, aiohttp_client, sentinel + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # view = yield table.view() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() - # size = yield table.size() + size = await table.size() - # assert size == 10 + assert size == 10 - # for i in range(5): - # yield table.make_port() + for i in range(5): + await table.make_port() - # port = yield table.make_port() + port = await table.make_port() - # s = sentinel(False) + s = sentinel(False) - # def updater(port_id, delta): - # s.set(True) - # t2 = Table(delta) - # assert t2.view().to_dict() == data - # assert port_id == port + def updater(port_id, delta): + s.set(True) + t2 = Table(delta) + assert t2.view().to_dict() == data + assert port_id == port - # view.on_update(updater, mode="row") + view.on_update(updater, mode="row") - # table.update(data, port_id=port) + table.update(data, port_id=port) - # size2 = yield table.size() - # assert size2 == 20 - # assert s.get() is True + size2 = await table.size() + assert size2 == 20 + assert s.get() is True - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_table_remove(self, app, http_client, http_port): - # table_name = str(random.random()) - # _table = Table(data, index="a") - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_table_remove(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data, index="a") + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # size = yield table.size() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + size = await table.size() - # assert size == 10 + assert size == 10 - # table.remove([i for i in range(5)]) + table.remove([i for i in range(5)]) - # view = yield table.view(columns=["a"]) - # output = yield view.to_dict() + view = await table.view(columns=["a"]) + output = await view.to_dict() - # assert output == {"a": [i for i in range(5, 10)]} + assert output == {"a": [i for i in range(5, 10)]} - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_create_view( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # view = yield table.view(columns=["a"]) - # output = yield view.to_dict() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view(columns=["a"]) + output = await view.to_dict() - # assert output == { - # "a": [i for i in range(10)], - # } + assert output == { + "a": [i for i in range(10)], + } - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_create_view_errors( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view_errors(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) - # with pytest.raises(PerspectiveError) as exc: - # yield table.view(columns=["abcde"]) + with pytest.raises(PerspectiveError) as exc: + await table.view(columns=["abcde"]) - # assert str(exc.value) == "Invalid column 'abcde' found in View columns.\n" + assert str(exc.value) == "Invalid column 'abcde' found in View columns.\n" - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_create_view_to_arrow( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view_to_arrow(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # view = yield table.view() - # output = yield view.to_arrow() - # expected = yield table.schema() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() + output = await view.to_arrow() + expected = await table.schema() - # assert Table(output).schema(as_string=True) == expected + assert Table(output).schema(as_string=True) == expected - # @pytest.mark.gen_test(run_sync=False) - # def test_tornado_handler_create_view_to_arrow_update( - # self, app, http_client, http_port, sentinel - # ): - # table_name = str(random.random()) - # _table = Table(data) - # MANAGER.host_table(table_name, _table) + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view_to_arrow_update( + self, app, aiohttp_client + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) - # client = yield self.websocket_client(http_port) - # table = client.open_table(table_name) - # view = yield table.view() + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() - # output = yield view.to_arrow() + output = await view.to_arrow() - # for i in range(10): - # table.update(output) + for i in range(10): + await table.update(output) - # size2 = yield table.size() - # assert size2 == 110 + size2 = await table.size() + assert size2 == 110 diff --git a/python/perspective/perspective/tests/handlers/test_aiohttp_handler_chunked.py b/python/perspective/perspective/tests/handlers/test_aiohttp_handler_chunked.py new file mode 100644 index 0000000000..c669af5fd7 --- /dev/null +++ b/python/perspective/perspective/tests/handlers/test_aiohttp_handler_chunked.py @@ -0,0 +1,126 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# + +# NOTE: This is essentially a clone of the tornado handler tests, +# but using async/await and starlette handler +import random +import pytest + +from aiohttp import web +from datetime import datetime + +from perspective import ( + Table, + PerspectiveManager, + PerspectiveAIOHTTPHandler, + aiohttp_websocket as websocket, +) + + +data = { + "a": [i for i in range(10)], + "b": [i * 1.5 for i in range(10)], + "c": [str(i) for i in range(10)], + "d": [datetime(2020, 3, i, i, 30, 45) for i in range(1, 11)], +} + +MANAGER = PerspectiveManager() + + +async def websocket_handler(request): + handler = PerspectiveAIOHTTPHandler( + manager=MANAGER, request=request, chunk_size=500 + ) + await handler.run() + + +@pytest.fixture +def app(): + app = web.Application() + app.router.add_get("/websocket", websocket_handler) + return app + + +class TestPerspectiveAIOHTTPHandlerChunked(object): + def setup_method(self): + """Flush manager state before each test method execution.""" + MANAGER._tables = {} + MANAGER._views = {} + + async def websocket_client(self, app, aiohttp_client): + """Connect and initialize a websocket client connection to the + Perspective aiottp server. + """ + client = await aiohttp_client(app) + return await websocket( + "http://{}:{}/websocket".format(client.host, client.port), client.session + ) + + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view_to_arrow_chunked( + self, app, aiohttp_client + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() + output = await view.to_arrow() + expected = await table.schema() + + assert Table(output).schema(as_string=True) == expected + + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view_to_arrow_update_chunked( + self, app, aiohttp_client + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() + + output = await view.to_arrow() + + for i in range(10): + await table.update(output) + + size2 = await table.size() + assert size2 == 110 + + @pytest.mark.asyncio + async def test_aiohttp_handler_update_chunked_interleaved_with_trivial( + self, app, aiohttp_client + ): + """Tests that, when a chunked response `output_fut` is interleaved with + a response belonging to another message ID (and not binary encoded) + `size3`, both messages de-multiplex correclty and succeed. + """ + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + view = await table.view() + + output_fut = view.to_arrow() + size3 = await view.num_rows() + assert size3 == 10 + + output = await output_fut + + for i in range(10): + await table.update(output) + + size2 = await table.size() + assert size2 == 110 diff --git a/python/perspective/perspective/tests/handlers/test_aiohttp_lock.py b/python/perspective/perspective/tests/handlers/test_aiohttp_lock.py new file mode 100644 index 0000000000..9b8c3f7a31 --- /dev/null +++ b/python/perspective/perspective/tests/handlers/test_aiohttp_lock.py @@ -0,0 +1,75 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# + +# NOTE: see test_tornado_lock for important notes +import asyncio +import pytest +import random + +from aiohttp import web +from datetime import datetime + +from perspective import ( + Table, + PerspectiveManager, + PerspectiveAIOHTTPHandler, + aiohttp_websocket as websocket, +) + + +data = { + "a": [i for i in range(10)], + "b": [i * 1.5 for i in range(10)], + "c": [str(i) for i in range(10)], + "d": [datetime(2020, 3, i, i, 30, 45) for i in range(1, 11)], +} + +MANAGER = PerspectiveManager() + + +async def websocket_handler(request): + handler = PerspectiveAIOHTTPHandler(manager=MANAGER, request=request, chunk_size=10) + await handler.run() + + +@pytest.fixture +def app(): + app = web.Application() + app.router.add_get("/websocket", websocket_handler) + return app + + +class TestPerspectiveAIOHTTPHandlerChunked(object): + def setup_method(self): + """Flush manager state before each test method execution.""" + MANAGER._tables = {} + MANAGER._views = {} + + async def websocket_client(self, app, aiohttp_client): + """Connect and initialize a websocket client connection to the + Perspective aiottp server. + """ + client = await aiohttp_client(app) + return await websocket( + "http://{}:{}/websocket".format(client.host, client.port), client.session + ) + + @pytest.mark.asyncio + async def test_aiohttp_handler_lock_inflight(self, app, aiohttp_client): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client(app, aiohttp_client) + table = client.open_table(table_name) + views = await asyncio.gather(*[table.view() for _ in range(5)]) + outputs = await asyncio.gather(*[view.to_arrow() for view in views]) + expected = await table.schema() + + for output in outputs: + assert Table(output).schema(as_string=True) == expected diff --git a/python/perspective/perspective/tests/handlers/test_starlette_handler.py b/python/perspective/perspective/tests/handlers/test_starlette_handler.py index 9b9275cb32..09ae06be54 100644 --- a/python/perspective/perspective/tests/handlers/test_starlette_handler.py +++ b/python/perspective/perspective/tests/handlers/test_starlette_handler.py @@ -17,9 +17,14 @@ from fastapi import FastAPI, WebSocket from fastapi.testclient import TestClient -from perspective import Table, PerspectiveManager, PerspectiveStarletteHandler, PerspectiveError +from perspective import ( + Table, + PerspectiveManager, + PerspectiveStarletteHandler, + PerspectiveError, +) -from ...client.starlette_test import websocket +from perspective.client.starlette_test import websocket data = { @@ -33,7 +38,7 @@ async def websocket_handler(websocket: WebSocket): - handler = PerspectiveStarletteHandler(MANAGER, websocket) + handler = PerspectiveStarletteHandler(manager=MANAGER, websocket=websocket) await handler.run() @@ -51,7 +56,7 @@ def setup_method(self): async def websocket_client(self): """Connect and initialize a websocket client connection to the - Perspective tornado server. + Perspective starlette server. """ return await websocket(CLIENT, "/websocket") @@ -213,7 +218,7 @@ async def test_starlette_handler_table_update_row_delta_port(self, sentinel): def updater(port_id, delta): s.set(True) assert port_id == port - + t2 = Table(delta) assert t2.view().to_dict() == data diff --git a/python/perspective/perspective/tests/handlers/test_starlette_handler_chunked.py b/python/perspective/perspective/tests/handlers/test_starlette_handler_chunked.py new file mode 100644 index 0000000000..f9c3a22eda --- /dev/null +++ b/python/perspective/perspective/tests/handlers/test_starlette_handler_chunked.py @@ -0,0 +1,123 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# + +# NOTE: This is essentially a clone of the tornado handler tests, +# but using async/await and starlette handler + +import pytest +import random + +from datetime import datetime + +from fastapi import FastAPI, WebSocket +from fastapi.testclient import TestClient + +from perspective import ( + Table, + PerspectiveManager, + PerspectiveStarletteHandler, +) + +from perspective.client.starlette_test import websocket + + +data = { + "a": [i for i in range(10)], + "b": [i * 1.5 for i in range(10)], + "c": [str(i) for i in range(10)], + "d": [datetime(2020, 3, i, i, 30, 45) for i in range(1, 11)], +} + +MANAGER = PerspectiveManager() + + +async def websocket_handler(websocket: WebSocket): + handler = PerspectiveStarletteHandler( + manager=MANAGER, websocket=websocket, chunk_size=500 + ) + await handler.run() + + +APPLICATION = FastAPI() +APPLICATION.add_api_websocket_route("/websocket", websocket_handler) + +CLIENT = TestClient(APPLICATION) + + +class TestPerspectiveStarletteHandlerChunked(object): + def setup_method(self): + """Flush manager state before each test method execution.""" + MANAGER._tables = {} + MANAGER._views = {} + + async def websocket_client(self): + """Connect and initialize a websocket client connection to the + Perspective starlette server. + """ + return await websocket(CLIENT, "/websocket") + + @pytest.mark.asyncio + async def test_starlette_handler_create_view_to_arrow_chunked(self): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client() + table = client.open_table(table_name) + view = await table.view() + output = await view.to_arrow() + expected = await table.schema() + + assert Table(output).schema(as_string=True) == expected + await client.terminate() + + @pytest.mark.asyncio + async def test_starlette_handler_create_view_to_arrow_update_chunked(self): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client() + table = client.open_table(table_name) + view = await table.view() + + output = await view.to_arrow() + + for i in range(10): + await table.update(output) + + size2 = await table.size() + assert size2 == 110 + await client.terminate() + + @pytest.mark.asyncio + async def test_starlette_handler_update_chunked_interleaved_with_trivial(self): + """Tests that, when a chunked response `output_fut` is interleaved with + a response belonging to another message ID (and not binary encoded) + `size3`, both messages de-multiplex correclty and succeed. + """ + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client() + table = client.open_table(table_name) + view = await table.view() + + output_fut = view.to_arrow() + size3 = await view.num_rows() + assert size3 == 10 + + output = await output_fut + + for i in range(10): + await table.update(output) + + size2 = await table.size() + assert size2 == 110 + await client.terminate() diff --git a/python/perspective/perspective/tests/handlers/test_starlette_lock.py b/python/perspective/perspective/tests/handlers/test_starlette_lock.py new file mode 100644 index 0000000000..11c6d8af43 --- /dev/null +++ b/python/perspective/perspective/tests/handlers/test_starlette_lock.py @@ -0,0 +1,91 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# + +# NOTE: see test_tornado_lock for important notes +import asyncio +import pytest +import random + +from datetime import datetime + +from fastapi import FastAPI, WebSocket +from fastapi.testclient import TestClient + +from perspective import ( + Table, + PerspectiveManager, + PerspectiveStarletteHandler, +) + +from perspective.client.starlette_test import websocket + + +data = { + "a": [i for i in range(10)], + "b": [i * 1.5 for i in range(10)], + "c": [str(i) for i in range(10)], + "d": [datetime(2020, 3, i, i, 30, 45) for i in range(1, 11)], +} + +MANAGER = PerspectiveManager() + + +async def websocket_handler(websocket: WebSocket): + handler = PerspectiveStarletteHandler( + manager=MANAGER, websocket=websocket, chunk_size=500 + ) + await handler.run() + + +APPLICATION = FastAPI() +APPLICATION.add_api_websocket_route("/websocket", websocket_handler) + +CLIENT = TestClient(APPLICATION) + + +class TestPerspectiveStarletteHandlerChunked(object): + def setup_method(self): + """Flush manager state before each test method execution.""" + MANAGER._tables = {} + MANAGER._views = {} + + async def websocket_client(self): + """Connect and initialize a websocket client connection to the + Perspective starlette server. + """ + return await websocket(CLIENT, "/websocket") + + @pytest.mark.asyncio + async def test_aiohttp_handler_create_view_to_arrow_chunked(self): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client() + table = client.open_table(table_name) + view = await table.view() + output = await view.to_arrow() + expected = await table.schema() + + assert Table(output).schema(as_string=True) == expected + await client.terminate() + + @pytest.mark.asyncio + async def test_starlette_handler_create_view_to_arrow_chunked(self): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client() + table = client.open_table(table_name) + views = await asyncio.gather(*[table.view() for _ in range(5)]) + outputs = await asyncio.gather(*[view.to_arrow() for view in views]) + expected = await table.schema() + + for output in outputs: + assert Table(output).schema(as_string=True) == expected diff --git a/python/perspective/perspective/tests/handlers/test_tornado_handler.py b/python/perspective/perspective/tests/handlers/test_tornado_handler.py index dc9de17bd7..5718c1653c 100644 --- a/python/perspective/perspective/tests/handlers/test_tornado_handler.py +++ b/python/perspective/perspective/tests/handlers/test_tornado_handler.py @@ -9,10 +9,15 @@ import pytest import tornado -from tornado import gen from datetime import datetime -from perspective import Table, PerspectiveManager, PerspectiveTornadoHandler, PerspectiveError, tornado_websocket as websocket +from perspective import ( + Table, + PerspectiveManager, + PerspectiveTornadoHandler, + PerspectiveError, + tornado_websocket as websocket, +) data = { @@ -50,11 +55,7 @@ async def websocket_client(self, port): """Connect and initialize a websocket client connection to the Perspective tornado server. """ - client = await websocket( - "ws://127.0.0.1:{0}/websocket".format(port) - ) - - # Compatibility with Python < 3.3 + client = await websocket("ws://127.0.0.1:{0}/websocket".format(port)) return client @pytest.mark.gen_test(run_sync=False) @@ -65,7 +66,7 @@ async def test_tornado_handler_init_terminate(self, app, http_client, http_port) All test methods must import `app`, `http_client`, and `http_port`, otherwise a mysterious timeout will occur.""" client = await self.websocket_client(http_port) - client.terminate() + await client.terminate() @pytest.mark.gen_test(run_sync=False) async def test_tornado_handler_table_method(self, app, http_client, http_port): @@ -305,7 +306,7 @@ async def test_tornado_handler_create_view_to_arrow_update( output = await view.to_arrow() for i in range(10): - table.update(output) + await table.update(output) size2 = await table.size() assert size2 == 110 diff --git a/python/perspective/perspective/tests/handlers/test_tornado_handler_chunked.py b/python/perspective/perspective/tests/handlers/test_tornado_handler_chunked.py index edd10fbdde..2519a4ec6f 100644 --- a/python/perspective/perspective/tests/handlers/test_tornado_handler_chunked.py +++ b/python/perspective/perspective/tests/handlers/test_tornado_handler_chunked.py @@ -12,9 +12,12 @@ from tornado import gen from datetime import datetime -from ...table import Table -from ...manager import PerspectiveManager -from ...handlers.tornado import PerspectiveTornadoHandler, websocket +from perspective import ( + Table, + PerspectiveManager, + PerspectiveTornadoHandler, + tornado_websocket as websocket, +) data = { @@ -48,54 +51,51 @@ def setup_method(self): MANAGER._tables = {} MANAGER._views = {} - @gen.coroutine - def websocket_client(self, port): + async def websocket_client(self, port): """Connect and initialize a websocket client connection to the Perspective tornado server. """ - client = yield websocket("ws://127.0.0.1:{0}/websocket".format(port)) - - # Compatibility with Python < 3.3 - raise gen.Return(client) + client = await websocket("ws://127.0.0.1:{}/websocket".format(port)) + return client @pytest.mark.gen_test(run_sync=False) - def test_tornado_handler_create_view_to_arrow_chunked( + async def test_tornado_handler_create_view_to_arrow_chunked( self, app, http_client, http_port, sentinel ): table_name = str(random.random()) _table = Table(data) MANAGER.host_table(table_name, _table) - client = yield self.websocket_client(http_port) + client = await self.websocket_client(http_port) table = client.open_table(table_name) - view = yield table.view() - output = yield view.to_arrow() - expected = yield table.schema() + view = await table.view() + output = await view.to_arrow() + expected = await table.schema() assert Table(output).schema(as_string=True) == expected @pytest.mark.gen_test(run_sync=False) - def test_tornado_handler_create_view_to_arrow_update_chunked( + async def test_tornado_handler_create_view_to_arrow_update_chunked( self, app, http_client, http_port, sentinel ): table_name = str(random.random()) _table = Table(data) MANAGER.host_table(table_name, _table) - client = yield self.websocket_client(http_port) + client = await self.websocket_client(http_port) table = client.open_table(table_name) - view = yield table.view() + view = await table.view() - output = yield view.to_arrow() + output = await view.to_arrow() for i in range(10): - table.update(output) + await table.update(output) - size2 = yield table.size() + size2 = await table.size() assert size2 == 110 @pytest.mark.gen_test(run_sync=False) - def test_tornado_handler_update_chunked_interleaved_with_trivial( + async def test_tornado_handler_update_chunked_interleaved_with_trivial( self, app, http_client, http_port, sentinel ): """Tests that, when a chunked response `output_fut` is interleaved with @@ -106,18 +106,18 @@ def test_tornado_handler_update_chunked_interleaved_with_trivial( _table = Table(data) MANAGER.host_table(table_name, _table) - client = yield self.websocket_client(http_port) + client = await self.websocket_client(http_port) table = client.open_table(table_name) - view = yield table.view() + view = await table.view() output_fut = view.to_arrow() - size3 = yield view.num_rows() + size3 = await view.num_rows() assert size3 == 10 - output = yield output_fut + output = await output_fut for i in range(10): - table.update(output) + await table.update(output) - size2 = yield table.size() + size2 = await table.size() assert size2 == 110 diff --git a/python/perspective/perspective/tests/handlers/test_tornado_lock.py b/python/perspective/perspective/tests/handlers/test_tornado_lock.py new file mode 100644 index 0000000000..11cd769a55 --- /dev/null +++ b/python/perspective/perspective/tests/handlers/test_tornado_lock.py @@ -0,0 +1,97 @@ +################################################################################ +# +# Copyright (c) 2019, the Perspective Authors. +# +# This file is part of the Perspective library, distributed under the terms of +# the Apache License 2.0. The full license can be found in the LICENSE file. +# + + +# This test demonstrates the necessity of locks for server +# responses to multiple clients + +# This test should be implemented for every new server handler, but +# should otherwise never fail for existing locking implementations. + +# To demonstrate failing behavior, modify the tornado handler like so: +# - yield self._stream_lock.acquire() +# + # yield self._stream_lock.acquire() +# try: +# yield f(*args, **kwargs) +# except tornado.websocket.WebSocketClosedError: +# pass +# finally: +# - yield self._stream_lock.release() +# + ... +# + # yield self._stream_lock.release() + + +import asyncio +import pytest +import random + +import tornado +from datetime import datetime + +from perspective import ( + Table, + PerspectiveManager, + PerspectiveTornadoHandler, + tornado_websocket as websocket, +) + + +data = { + "a": [i for i in range(10)], + "b": [i * 1.5 for i in range(10)], + "c": [str(i) for i in range(10)], + "d": [datetime(2020, 3, i, i, 30, 45) for i in range(1, 11)], +} + +MANAGER = PerspectiveManager() + +APPLICATION = tornado.web.Application( + [ + ( + r"/websocket", + PerspectiveTornadoHandler, + {"manager": MANAGER, "check_origin": True, "chunk_size": 10}, + ) + ] +) + + +@pytest.fixture +def app(): + return APPLICATION + + +class TestPerspectiveTornadoHandlerChunked(object): + def setup_method(self): + """Flush manager state before each test method execution.""" + MANAGER._tables = {} + MANAGER._views = {} + + async def websocket_client(self, port): + """Connect and initialize a websocket client connection to the + Perspective tornado server. + """ + client = await websocket("ws://127.0.0.1:{}/websocket".format(port)) + return client + + @pytest.mark.gen_test(run_sync=False) + async def test_tornado_handler_lock_inflight( + self, app, http_client, http_port, sentinel + ): + table_name = str(random.random()) + _table = Table(data) + MANAGER.host_table(table_name, _table) + + client = await self.websocket_client(http_port) + table = client.open_table(table_name) + views = await asyncio.gather(*[table.view() for _ in range(5)]) + outputs = await asyncio.gather(*[view.to_arrow() for view in views]) + expected = await table.schema() + + for output in outputs: + assert Table(output).schema(as_string=True) == expected diff --git a/python/perspective/perspective/tests/manager/__init__.py b/python/perspective/perspective/tests/manager/__init__.py index b4395b8093..42db35c427 100644 --- a/python/perspective/perspective/tests/manager/__init__.py +++ b/python/perspective/perspective/tests/manager/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/tests/table/__init__.py b/python/perspective/perspective/tests/table/__init__.py index 4cb669574b..42db35c427 100644 --- a/python/perspective/perspective/tests/table/__init__.py +++ b/python/perspective/perspective/tests/table/__init__.py @@ -4,4 +4,4 @@ # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. -# \ No newline at end of file +# diff --git a/python/perspective/perspective/tests/table/test_table_pandas.py b/python/perspective/perspective/tests/table/test_table_pandas.py index 872e5fe277..aba6ca511a 100644 --- a/python/perspective/perspective/tests/table/test_table_pandas.py +++ b/python/perspective/perspective/tests/table/test_table_pandas.py @@ -12,8 +12,6 @@ import pandas as pd from perspective.table import Table -from ..common import superstore - class TestTablePandas(object): def test_empty_table(self): @@ -886,18 +884,16 @@ def test_table_indexed_series(self): } assert tbl.size() == 3 - def test_groupbys(self): - df = superstore(100) - df_pivoted = df.set_index(['Country', 'Region']) + def test_groupbys(self, superstore): + df_pivoted = superstore.set_index(['Country', 'Region']) table = Table(df_pivoted) columns = table.columns() assert table.size() == 100 assert "Country" in columns assert "Region" in columns - def test_pivottable(self): - df = superstore() - pt = pd.pivot_table(df, values='Discount', index=['Country', 'Region'], columns='Category') + def test_pivottable(self, superstore): + pt = pd.pivot_table(superstore, values='Discount', index=['Country', 'Region'], columns='Category') table = Table(pt) columns = table.columns() assert "Country" in columns diff --git a/python/perspective/perspective/tests/viewer/__init__.py b/python/perspective/perspective/tests/viewer/__init__.py index b4395b8093..42db35c427 100644 --- a/python/perspective/perspective/tests/viewer/__init__.py +++ b/python/perspective/perspective/tests/viewer/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/tests/widget/__init__.py b/python/perspective/perspective/tests/widget/__init__.py index b4395b8093..42db35c427 100644 --- a/python/perspective/perspective/tests/widget/__init__.py +++ b/python/perspective/perspective/tests/widget/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (c) 2022, the Perspective Authors. +# Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/python/perspective/perspective/tests/widget/test_widget_pandas.py b/python/perspective/perspective/tests/widget/test_widget_pandas.py index bc6c9f238d..d3d7051245 100644 --- a/python/perspective/perspective/tests/widget/test_widget_pandas.py +++ b/python/perspective/perspective/tests/widget/test_widget_pandas.py @@ -10,15 +10,12 @@ import pandas as pd import numpy as np from perspective import Table, PerspectiveWidget -from ..common import superstore - -DF = superstore(200) class TestWidgetPandas: - def test_widget_load_table_df(self): - table = Table(DF) + def test_widget_load_table_df(self, superstore): + table = Table(superstore) widget = PerspectiveWidget(table) assert widget.table.schema() == {'index': int, 'Country': str, 'Region': str, 'Category': str, 'City': str, 'Customer ID': str, 'Discount': float, 'Order Date': date, 'Order ID': str, 'Postal Code': str, 'Product ID': str, 'Profit': float, 'Quantity': int, @@ -28,30 +25,30 @@ def test_widget_load_table_df(self): 'Product ID', 'Profit', 'Quantity', 'Region', 'Row ID', 'Sales', 'Segment', 'Ship Date', 'Ship Mode', 'State', 'Sub-Category']) view = widget.table.view() - assert view.num_rows() == len(DF) - assert view.num_columns() == len(DF.columns) + 1 # index + assert view.num_rows() == len(superstore) + assert view.num_columns() == len(superstore.columns) + 1 # index - def test_widget_load_data_df(self): - widget = PerspectiveWidget(DF) + def test_widget_load_data_df(self, superstore): + widget = PerspectiveWidget(superstore) assert sorted(widget.columns) == sorted(['index', 'Category', 'City', 'Country', 'Customer ID', 'Discount', 'Order Date', 'Order ID', 'Postal Code', 'Product ID', 'Profit', 'Quantity', 'Region', 'Row ID', 'Sales', 'Segment', 'Ship Date', 'Ship Mode', 'State', 'Sub-Category']) view = widget.table.view() - assert view.num_rows() == len(DF) + assert view.num_rows() == len(superstore) assert view.num_columns() == 20 - def test_widget_load_series(self): - series = pd.Series(DF["Profit"].values, name="profit") + def test_widget_load_series(self, superstore): + series = pd.Series(superstore["Profit"].values, name="profit") widget = PerspectiveWidget(series) assert widget.table.schema() == {'index': int, 'profit': float} assert sorted(widget.columns) == sorted(["index", "profit"]) view = widget.table.view() - assert view.num_rows() == len(DF) + assert view.num_rows() == len(superstore) assert view.num_columns() == 2 - def test_widget_load_pivot_table(self): - pivot_table = pd.pivot_table(DF, values='Discount', index=['Country', 'Region'], columns=['Category', 'Segment']) + def test_widget_load_pivot_table(self, superstore): + pivot_table = pd.pivot_table(superstore, values='Discount', index=['Country', 'Region'], columns=['Category', 'Segment']) widget = PerspectiveWidget(pivot_table) assert widget.group_by == ['Country', 'Region'] assert widget.split_by == ['Category', 'Segment'] @@ -61,8 +58,8 @@ def test_widget_load_pivot_table(self): assert view.num_rows() == 60 assert view.num_columns() == 6 - def test_widget_load_pivot_table_with_user_pivots(self): - pivot_table = pd.pivot_table(DF, values='Discount', index=['Country', 'Region'], columns='Category') + def test_widget_load_pivot_table_with_user_pivots(self, superstore): + pivot_table = pd.pivot_table(superstore, values='Discount', index=['Country', 'Region'], columns='Category') widget = PerspectiveWidget(pivot_table, group_by=["Category", "Segment"]) assert widget.group_by == ['Category', 'Segment'] assert widget.split_by == [] @@ -72,33 +69,33 @@ def test_widget_load_pivot_table_with_user_pivots(self): assert view.num_rows() == 5 assert view.num_columns() == 6 - def test_widget_load_group_by(self): - df_pivoted = DF.set_index(['Country', 'Region']) + def test_widget_load_group_by(self, superstore): + df_pivoted = superstore.set_index(['Country', 'Region']) widget = PerspectiveWidget(df_pivoted) assert widget.group_by == ['Country', 'Region'] assert widget.split_by == [] assert sorted(widget.columns) == sorted(['index', 'Category', 'Country', 'City', 'Customer ID', 'Discount', 'Order Date', 'Order ID', 'Postal Code', 'Product ID', 'Profit', 'Quantity', 'Region', 'Row ID', 'Sales', 'Segment', 'Ship Date', 'Ship Mode', 'State', 'Sub-Category']) - assert widget.table.size() == 200 + assert widget.table.size() == 100 view = widget.table.view() - assert view.num_rows() == len(DF) - assert view.num_columns() == len(DF.columns) + 1 # index + assert view.num_rows() == len(superstore) + assert view.num_columns() == len(superstore.columns) + 1 # index - def test_widget_load_group_by_with_user_pivots(self): - df_pivoted = DF.set_index(['Country', 'Region']) + def test_widget_load_group_by_with_user_pivots(self, superstore): + df_pivoted = superstore.set_index(['Country', 'Region']) widget = PerspectiveWidget(df_pivoted, group_by=["Category", "Segment"]) assert widget.group_by == ['Category', 'Segment'] assert widget.split_by == [] assert sorted(widget.columns) == sorted(['index', 'Category', 'Country', 'City', 'Customer ID', 'Discount', 'Order Date', 'Order ID', 'Postal Code', 'Product ID', 'Profit', 'Quantity', 'Region', 'Row ID', 'Sales', 'Segment', 'Ship Date', 'Ship Mode', 'State', 'Sub-Category']) - assert widget.table.size() == 200 + assert widget.table.size() == 100 view = widget.table.view() - assert view.num_rows() == len(DF) - assert view.num_columns() == len(DF.columns) + 1 # index + assert view.num_rows() == len(superstore) + assert view.num_columns() == len(superstore.columns) + 1 # index - def test_widget_load_split_by(self): + def test_widget_load_split_by(self, superstore): arrays = [np.array(['bar', 'bar', 'bar', 'bar', 'baz', 'baz', 'baz', 'baz', 'foo', 'foo', 'foo', 'foo', 'qux', 'qux', 'qux', 'qux']), np.array(['one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two']), np.array(['X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y'])] @@ -110,7 +107,7 @@ def test_widget_load_split_by(self): assert widget.split_by == ['first', 'second', 'third'] assert widget.group_by == ['index'] - def test_widget_load_split_by_preserve_user_settings(self): + def test_widget_load_split_by_preserve_user_settings(self, superstore): arrays = [np.array(['bar', 'bar', 'bar', 'bar', 'baz', 'baz', 'baz', 'baz', 'foo', 'foo', 'foo', 'foo', 'qux', 'qux', 'qux', 'qux']), np.array(['one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two']), np.array(['X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y'])] @@ -122,7 +119,7 @@ def test_widget_load_split_by_preserve_user_settings(self): assert widget.split_by == ['first', 'second', 'third'] assert widget.group_by == ['index'] - def test_pivottable_values_index(self): + def test_pivottable_values_index(self, superstore): arrays = {'A':['bar', 'bar', 'bar', 'bar', 'baz', 'baz', 'baz', 'baz', 'foo', 'foo', 'foo', 'foo', 'qux', 'qux', 'qux', 'qux'], 'B':['one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two', 'one', 'one', 'two', 'two'], 'C':['X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y', 'X', 'Y'], @@ -135,8 +132,8 @@ def test_pivottable_values_index(self): assert widget.split_by == ['B', 'C'] assert widget.group_by == ['A'] - def test_pivottable_multi_values(self): - pt = pd.pivot_table(DF, values = ['Discount','Sales'], index=['Country','Region'],aggfunc={'Discount':'count','Sales':'sum'},columns=["State","Quantity"]) + def test_pivottable_multi_values(self, superstore): + pt = pd.pivot_table(superstore, values = ['Discount','Sales'], index=['Country','Region'],aggfunc={'Discount':'count','Sales':'sum'},columns=["State","Quantity"]) widget = PerspectiveWidget(pt) assert widget.columns == ['Discount', 'Sales'] assert widget.split_by == ['State', 'Quantity'] diff --git a/python/perspective/setup.cfg b/python/perspective/setup.cfg index 625d9ec926..8881fe2791 100644 --- a/python/perspective/setup.cfg +++ b/python/perspective/setup.cfg @@ -15,3 +15,8 @@ exclude=perspective/tests/ per-file-ignores = __init__.py: F401, F403 libpsp.py: F401, F403 + +[tool:pytest] +asyncio_mode=strict +testpaths = + perspective/tests \ No newline at end of file diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 6feb72c196..e16260a36f 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -60,9 +60,11 @@ def get_version(file, name="__version__"): "traitlets>=4.3.2", ] -requires_tornado = ["tornado>=4.5.3"] +requires_aiohttp = ["aiohttp"] + +requires_starlette = ["fastapi", "starlette"] -requires_starlette = ["aiohttp", "fastapi", "starlette"] +requires_tornado = ["tornado>=4.5.3"] requires_dev = ( [ @@ -75,6 +77,7 @@ def get_version(file, name="__version__"): "pybind11>=2.4.0", "pyarrow>=0.16.0", "pytest>=4.3.0", + "pytest-aiohttp", "pytest-asyncio", "pytest-cov>=2.6.1", "pytest-check-links", @@ -85,8 +88,9 @@ def get_version(file, name="__version__"): "wheel", ] + requires - + requires_tornado + + requires_aiohttp + requires_starlette + + requires_tornado ) @@ -271,7 +275,7 @@ def run_check(self): python_requires=">=3.6", install_requires=requires, extras_require={ - "aiohttp": requires_starlette, + "aiohttp": requires_aiohttp, "dev": requires_dev, "fastapi": requires_starlette, "starlette": requires_starlette,