From 2d305112498db3718fe8720d69c71917ba8fa921 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Mon, 4 Jul 2022 07:49:14 +0200 Subject: [PATCH 001/103] wip app pull state --- .../add_web_ui/panel/app_pull_state.py | 66 +++++++++++++++++++ .../add_web_ui/panel/intermediate.rst | 18 +++++ .../add_web_ui/panel/panel_frontend.py | 52 +++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 docs/source-app/workflows/add_web_ui/panel/app_pull_state.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/intermediate.rst create mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_frontend.py diff --git a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py new file mode 100644 index 0000000000000..ba0f361634825 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py @@ -0,0 +1,66 @@ +"""This app demonstrates how the PanelFrontend can read/ pull the global app +state and display it""" +import datetime + +import lightning as L +import panel as pn +from panel_frontend import PanelFrontend +from pathlib import Path +import json + +DATETIME_FORMAT = "%Y-%m-%d, %H:%M:%S.%f" + +APP_STATE = Path(__file__).parent / "app_state.json" + +def read_app_state(): + with open(APP_STATE) as json_file: + return json.load(json_file) + +def save_app_state(state): + with open(APP_STATE, 'w') as outfile: + json.dump(state, outfile) + +def to_string(value: datetime.datetime)->str: + return value.strftime(DATETIME_FORMAT) + +def render_fn(self): + global_app_state_pane = pn.pane.JSON(depth=2) + last_local_update_pane = pn.pane.Str() + + def update(): + last_local_update_pane.object= "last local update: " + to_string(datetime.datetime.now()) + + # Todo: Figure out the right way to read/ pull the app state + global_app_state_pane.object = read_app_state() + + # Todo: Figure out how to schedule the callback globally to make the app scale + # There is no reason that each session should read or pull into memory individually + pn.state.add_periodic_callback(update, period=1000) + # Todo: Refactor the Panel app implementation to a more reactive api + # Todo: Giver the Panel app a nicer UX + return pn.Column(last_local_update_pane, global_app_state_pane) + +class Flow(L.LightningFlow): + def __init__(self): + super().__init__() + + self.panel_frontend = PanelFrontend(render_fn=render_fn) + self._last_global_update = datetime.datetime.now() + self.last_global_update = to_string(self._last_global_update) + + def run(self): + self.panel_frontend.run() + now = datetime.datetime.now() + if (now-self._last_global_update).microseconds>=100: + save_app_state(self.state) + self._last_global_update=now + self.last_global_update = to_string(now) + + def configure_layout(self): + tab1 = {"name": "Home", "content": self.panel_frontend} + return tab1 + +app = L.LightningApp(Flow()) + +if __name__.startswith("bokeh"): + render_fn(None).servable() \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst new file mode 100644 index 0000000000000..add90d1d233d7 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -0,0 +1,18 @@ +###################################### +Add a web UI with Panel (intermediate) +###################################### + +**Audience:** Users who want to communicate between the Lightning App and Panel. + +**Prereqs:** Must have read the `panel basic `_ guide. + +---- + +************************ +Display Global App State +************************ + +The `PanelFrontend` enables you read/pull and display (parts of) the global +app state. + +See app_pull_state.py. \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py new file mode 100644 index 0000000000000..23062c4219ce0 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py @@ -0,0 +1,52 @@ +import logging +import sys + +import lightning_app as lapp +import panel as pn +from lightning_app.utilities.imports import requires + +logger = logging.getLogger("PanelFrontend") +logger.setLevel(logging.DEBUG) + +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter('%(asctime)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) + +class PanelFrontend(lapp.LightningWork): + + @requires("panel") + def __init__(self, render_fn, parallel=True, **kwargs): + super().__init__(parallel=parallel, **kwargs) + self._render_fn = render_fn + self._server = None + self.requests = 0 + logger.info("init finished") + + def _fast_initial_view(self): + self.requests += 1 + logger.info("Session %s started", self.requests) + if self.requests == 1: + return pn.pane.HTML("

Please refresh the browser to see the app.

") + else: + return self._render_fn(self) + + def run(self): + logger.info("run start") + if not self._server: + logger.info("LitPanel Starting Server") + self._server = pn.serve( + {"/": self._fast_initial_view}, + port=self.port, + address=self.host, + websocket_origin="*", + show=False, + # threaded=True, + ) + logger.info("run end") + + def stop(self): + """Stops the server""" + if self._server: + self._server.stop() + logger.info("stop end") \ No newline at end of file From 50f1e2a64bf4dc477944ecb1b1a452820871dd5a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Jul 2022 05:53:28 +0000 Subject: [PATCH 002/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../add_web_ui/panel/app_pull_state.py | 34 +++++++++++-------- .../add_web_ui/panel/intermediate.rst | 2 +- .../add_web_ui/panel/panel_frontend.py | 11 +++--- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py index ba0f361634825..0a6cb83755938 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py @@ -1,35 +1,39 @@ -"""This app demonstrates how the PanelFrontend can read/ pull the global app -state and display it""" +"""This app demonstrates how the PanelFrontend can read/ pull the global app state and display it.""" import datetime +import json +from pathlib import Path -import lightning as L import panel as pn from panel_frontend import PanelFrontend -from pathlib import Path -import json + +import lightning as L DATETIME_FORMAT = "%Y-%m-%d, %H:%M:%S.%f" APP_STATE = Path(__file__).parent / "app_state.json" + def read_app_state(): with open(APP_STATE) as json_file: return json.load(json_file) + def save_app_state(state): - with open(APP_STATE, 'w') as outfile: + with open(APP_STATE, "w") as outfile: json.dump(state, outfile) -def to_string(value: datetime.datetime)->str: + +def to_string(value: datetime.datetime) -> str: return value.strftime(DATETIME_FORMAT) + def render_fn(self): global_app_state_pane = pn.pane.JSON(depth=2) last_local_update_pane = pn.pane.Str() - + def update(): - last_local_update_pane.object= "last local update: " + to_string(datetime.datetime.now()) - + last_local_update_pane.object = "last local update: " + to_string(datetime.datetime.now()) + # Todo: Figure out the right way to read/ pull the app state global_app_state_pane.object = read_app_state() @@ -40,10 +44,11 @@ def update(): # Todo: Giver the Panel app a nicer UX return pn.Column(last_local_update_pane, global_app_state_pane) + class Flow(L.LightningFlow): def __init__(self): super().__init__() - + self.panel_frontend = PanelFrontend(render_fn=render_fn) self._last_global_update = datetime.datetime.now() self.last_global_update = to_string(self._last_global_update) @@ -51,16 +56,17 @@ def __init__(self): def run(self): self.panel_frontend.run() now = datetime.datetime.now() - if (now-self._last_global_update).microseconds>=100: + if (now - self._last_global_update).microseconds >= 100: save_app_state(self.state) - self._last_global_update=now + self._last_global_update = now self.last_global_update = to_string(now) def configure_layout(self): tab1 = {"name": "Home", "content": self.panel_frontend} return tab1 + app = L.LightningApp(Flow()) if __name__.startswith("bokeh"): - render_fn(None).servable() \ No newline at end of file + render_fn(None).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index add90d1d233d7..03fb9bb7bca23 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -15,4 +15,4 @@ Display Global App State The `PanelFrontend` enables you read/pull and display (parts of) the global app state. -See app_pull_state.py. \ No newline at end of file +See app_pull_state.py. diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py index 23062c4219ce0..6859fc5dec441 100644 --- a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py @@ -1,20 +1,21 @@ import logging import sys -import lightning_app as lapp import panel as pn + +import lightning_app as lapp from lightning_app.utilities.imports import requires logger = logging.getLogger("PanelFrontend") logger.setLevel(logging.DEBUG) handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter('%(asctime)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) + class PanelFrontend(lapp.LightningWork): - @requires("panel") def __init__(self, render_fn, parallel=True, **kwargs): super().__init__(parallel=parallel, **kwargs) @@ -46,7 +47,7 @@ def run(self): logger.info("run end") def stop(self): - """Stops the server""" + """Stops the server.""" if self._server: self._server.stop() - logger.info("stop end") \ No newline at end of file + logger.info("stop end") From 3a22d2f97a1858e274b8a1245b118f0cc0d0fb7e Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Thu, 7 Jul 2022 19:45:36 +0200 Subject: [PATCH 003/103] basic principle --- .../workflows/add_web_ui/panel/app.py | 42 +++++++++++ .../add_web_ui/panel/app_pull_state.py | 2 +- .../add_web_ui/panel/app_streamlit.py | 22 ++++++ .../add_web_ui/panel/panel_frontend.py | 73 ++++++++++--------- .../add_web_ui/panel/panel_plugin.py | 65 +++++++++++++++++ .../add_web_ui/panel/panel_serve_render_fn.py | 66 +++++++++++++++++ .../workflows/add_web_ui/panel/test_panel.py | 16 ++++ 7 files changed, 252 insertions(+), 34 deletions(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/app.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/app_streamlit.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_plugin.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/test_panel.py diff --git a/docs/source-app/workflows/add_web_ui/panel/app.py b/docs/source-app/workflows/add_web_ui/panel/app.py new file mode 100644 index 0000000000000..cf8657bbdfc45 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/app.py @@ -0,0 +1,42 @@ +# app.py +import time +import lightning as L +import panel as pn +from panel_frontend import PanelFrontend +from lightning_app.utilities.state import AppState +import datetime as dt + +def your_panel_app(lightning_app_state: AppState): + return pn.Column( + pn.pane.Markdown("hello"), + lightning_app_state._plugin.param_state.param.value + ) + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend(render_fn=your_panel_app) + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.last_update = dt.datetime.now().isoformat() + self.counter = 0 + self.lit_panel = LitPanel() + + def run(self, *args, **kwargs) -> None: + time.sleep(0.1) + if self.counter<2000: + self.last_update = dt.datetime.now().isoformat() + self.counter += 1 + print(self.counter, self.last_update) + return super().run(*args, **kwargs) + + def configure_layout(self): + tab1 = {"name": "home", "content": self.lit_panel} + return tab1 + +app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py index ba0f361634825..2a990bc104f2c 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py @@ -38,7 +38,7 @@ def update(): pn.state.add_periodic_callback(update, period=1000) # Todo: Refactor the Panel app implementation to a more reactive api # Todo: Giver the Panel app a nicer UX - return pn.Column(last_local_update_pane, global_app_state_pane) + return pn.Column(last_local_update_pane, "## Global App State", global_app_state_pane) class Flow(L.LightningFlow): def __init__(self): diff --git a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py b/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py new file mode 100644 index 0000000000000..da56cdfbb0a28 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py @@ -0,0 +1,22 @@ +# app.py +import lightning as L +from lightning.app.frontend.stream_lit import StreamlitFrontend +import streamlit as st + +def your_streamlit_app(lightning_app_state): + st.write('hello world') + +class LitStreamlit(L.LightningFlow): + def configure_layout(self): + return StreamlitFrontend(render_fn=your_streamlit_app) + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_streamlit = LitStreamlit() + + def configure_layout(self): + tab1 = {"name": "home", "content": self.lit_streamlit} + return tab1 + +app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py index 23062c4219ce0..1540953f1889f 100644 --- a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py @@ -1,9 +1,14 @@ +import inspect import logging +import os +import subprocess import sys +from typing import Callable -import lightning_app as lapp -import panel as pn +from lightning_app.frontend.frontend import Frontend from lightning_app.utilities.imports import requires +from lightning_app.utilities.log import get_frontend_logfile +import pathlib logger = logging.getLogger("PanelFrontend") logger.setLevel(logging.DEBUG) @@ -13,40 +18,42 @@ handler.setFormatter(formatter) logger.addHandler(handler) -class PanelFrontend(lapp.LightningWork): +class PanelFrontend(Frontend): @requires("panel") - def __init__(self, render_fn, parallel=True, **kwargs): - super().__init__(parallel=parallel, **kwargs) - self._render_fn = render_fn - self._server = None - self.requests = 0 + def __init__(self, render_fn: Callable): # Would like to accept a `render_file` arguemnt too in the future + super().__init__() + + if inspect.ismethod(render_fn): + raise TypeError( + "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a pure function." + ) + + self.render_fn = render_fn logger.info("init finished") - def _fast_initial_view(self): - self.requests += 1 - logger.info("Session %s started", self.requests) - if self.requests == 1: - return pn.pane.HTML("

Please refresh the browser to see the app.

") - else: - return self._render_fn(self) - - def run(self): - logger.info("run start") - if not self._server: - logger.info("LitPanel Starting Server") - self._server = pn.serve( - {"/": self._fast_initial_view}, - port=self.port, - address=self.host, - websocket_origin="*", - show=False, - # threaded=True, + def start_server(self, host: str, port: int) -> None: + logger.info("starting server %s %s", host, port) + env = os.environ.copy() + env["LIGHTNING_FLOW_NAME"] = self.flow.name + env["LIGHTNING_RENDER_FUNCTION"] = self.render_fn.__name__ + env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(self.render_fn).__file__ + env["LIGHTNING_RENDER_PORT"] = str(port) + env["LIGHTNING_RENDER_ADDRESS"] = str(host) + std_err_out = get_frontend_logfile("error.log") + std_out_out = get_frontend_logfile("output.log") + with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: + self._process = subprocess.Popen( + [ + sys.executable, + pathlib.Path(__file__).parent / "panel_serve_render_fn.py", + ], + env=env, + # stdout=stdout, + # stderr=stderr, ) - logger.info("run end") - def stop(self): - """Stops the server""" - if self._server: - self._server.stop() - logger.info("stop end") \ No newline at end of file + def stop_server(self) -> None: + if self._process is None: + raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") + self._process.kill() \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_plugin.py b/docs/source-app/workflows/add_web_ui/panel/panel_plugin.py new file mode 100644 index 0000000000000..10034c9d2c993 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/panel_plugin.py @@ -0,0 +1,65 @@ +import asyncio +import logging +import os +import sys +import threading +import time + +import param +import websockets + +from lightning_app.core.constants import APP_SERVER_PORT +from lightning_app.utilities.app_helpers import AppStateType, BaseStatePlugin + +logger = logging.getLogger("PanelPlugin") +logger.setLevel(logging.DEBUG) + +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter('%(asctime)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) + +class ParamState(param.Parameterized): + value: int = param.Integer() + +def target_fn(param_state: ParamState): + async def update_fn(param_state: ParamState): + url = "localhost:8080" if "LIGHTNING_APP_STATE_URL" in os.environ else f"localhost:{APP_SERVER_PORT}" + ws_url = f"ws://{url}/api/v1/ws" + last_updated = time.time() + logger.info("connecting to web sockets %s", ws_url) + async with websockets.connect(ws_url) as websocket: + while True: + await websocket.recv() + while (time.time() - last_updated) < 1: + time.sleep(0.1) + last_updated = time.time() + param_state.value += 1 + logger.info("App state changed") + + asyncio.run(update_fn(param_state)) + + + +class PanelStatePlugin(BaseStatePlugin): + def __init__(self): + super().__init__() + import panel as pn + self.param_state = ParamState(value=1) + + if "_lightning_websocket_thread" not in pn.state.cache: + logger.info("starting thread") + thread = threading.Thread(target=target_fn, args=(self.param_state, )) + pn.state.cache["_lightning_websocket_thread"] = thread + thread.setDaemon(True) + thread.start() + logger.info("thread started") + + def should_update_app(self, deep_diff): + return deep_diff + + def get_context(self): + return {"type": AppStateType.DEFAULT.value} + + def render_non_authorized(self): + pass diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py new file mode 100644 index 0000000000000..c4794ff8748b5 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py @@ -0,0 +1,66 @@ +"""This file gets run by streamlit, which we launch within Lightning. + +From here, we will call the render function that the user provided in ``configure_layout``. +""" +import logging +import os +import pydoc +import sys +from typing import Callable, Union + +import panel as pn + +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState +from panel_plugin import PanelStatePlugin + +logger = logging.getLogger("PanelFrontend") +logger.setLevel(logging.DEBUG) + +handler = logging.StreamHandler(sys.stdout) +formatter = logging.Formatter('%(asctime)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) + +logger.info("starting plugin") +app_state = AppState(plugin=PanelStatePlugin()) # +logger.info("plugin started") + +def _get_render_fn_from_environment() -> Callable: + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + module = pydoc.importfile(render_fn_module_file) + return getattr(module, render_fn_name) + + +def _app_state_to_flow_scope(state: AppState, flow: Union[str, LightningFlow]) -> AppState: + """Returns a new AppState with the scope reduced to the given flow, as if the given flow as the root.""" + flow_name = flow.name if isinstance(flow, LightningFlow) else flow + flow_name_parts = flow_name.split(".")[1:] # exclude root + flow_state = state + for part in flow_name_parts: + flow_state = getattr(flow_state, part) + return flow_state + + +def view(): + flow_state = _app_state_to_flow_scope(app_state, flow=os.environ["LIGHTNING_FLOW_NAME"]) + render_fn = _get_render_fn_from_environment() + + return render_fn(app_state) + +def main(): + logger.info("Panel server starting") + port=int(os.environ["LIGHTNING_RENDER_PORT"]) + address=os.environ["LIGHTNING_RENDER_ADDRESS"] + url = os.environ["LIGHTNING_FLOW_NAME"] + pn.serve({url: view}, address=address, port=port, websocket_origin="*", show=False) + logger.info("Panel server started on port %s:%s/%s", address, port, url) + +# os.environ['LIGHTNING_FLOW_NAME']= 'root.lit_panel' +# os.environ['LIGHTNING_RENDER_FUNCTION']= 'your_panel_app' +# os.environ['LIGHTNING_RENDER_MODULE_FILE']= 'C:\\repos\\private\\lightning\\docs\\source-app\\workflows\\add_web_ui\\panel\\app.py' +# os.environ['LIGHTNING_RENDER_ADDRESS']= 'localhost' +# os.environ['LIGHTNING_RENDER_PORT']= '61965' + +main() \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/test_panel.py b/docs/source-app/workflows/add_web_ui/panel/test_panel.py new file mode 100644 index 0000000000000..2dc9717c19971 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/test_panel.py @@ -0,0 +1,16 @@ +import logging +import os +import pydoc +import sys +from typing import Callable, Union + +import panel as pn + +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState +from panel_plugin import PanelStatePlugin + + +def test_param_state(): + app_state = AppState(plugin=PanelStatePlugin()) # + assert hasattr(app_state._plugin.param_state) From cd133c68a296a32a65efcb026b7304ed3b93b5b4 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 8 Jul 2022 07:58:36 +0200 Subject: [PATCH 004/103] general principle working --- .../workflows/add_web_ui/panel/app.py | 42 ---------- .../workflows/add_web_ui/panel/app_basic.py | 26 +++++++ .../panel/app_interact_from_component.py | 50 ++++++++++++ .../panel/app_interact_from_frontend.py | 43 +++++++++++ .../add_web_ui/panel/app_pull_state.py | 72 ------------------ .../add_web_ui/panel/app_streamlit.py | 47 +++++++----- .../add_web_ui/panel/panel_frontend.py | 12 +-- .../add_web_ui/panel/panel_plugin.py | 65 ---------------- .../workflows/add_web_ui/panel/panel_serve.py | 45 +++++++++++ .../add_web_ui/panel/panel_serve_render_fn.py | 66 ---------------- .../workflows/add_web_ui/panel/panel_utils.py | 76 +++++++++++++++++++ .../workflows/add_web_ui/panel/test_panel.py | 32 ++++---- 12 files changed, 285 insertions(+), 291 deletions(-) delete mode 100644 docs/source-app/workflows/add_web_ui/panel/app.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/app_basic.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/app_pull_state.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_plugin.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_serve.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_utils.py diff --git a/docs/source-app/workflows/add_web_ui/panel/app.py b/docs/source-app/workflows/add_web_ui/panel/app.py deleted file mode 100644 index cf8657bbdfc45..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/app.py +++ /dev/null @@ -1,42 +0,0 @@ -# app.py -import time -import lightning as L -import panel as pn -from panel_frontend import PanelFrontend -from lightning_app.utilities.state import AppState -import datetime as dt - -def your_panel_app(lightning_app_state: AppState): - return pn.Column( - pn.pane.Markdown("hello"), - lightning_app_state._plugin.param_state.param.value - ) - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend(render_fn=your_panel_app) - - def configure_layout(self): - return self._frontend - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.last_update = dt.datetime.now().isoformat() - self.counter = 0 - self.lit_panel = LitPanel() - - def run(self, *args, **kwargs) -> None: - time.sleep(0.1) - if self.counter<2000: - self.last_update = dt.datetime.now().isoformat() - self.counter += 1 - print(self.counter, self.last_update) - return super().run(*args, **kwargs) - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_panel} - return tab1 - -app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/app_basic.py new file mode 100644 index 0000000000000..790ff46e66dcd --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/app_basic.py @@ -0,0 +1,26 @@ +# app.py +import lightning as L +import panel as pn +from panel_frontend import PanelFrontend +from panel_utils import AppStateWatcher + +def your_panel_app(app: AppStateWatcher): + return pn.pane.Markdown("hello") + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend(render_fn=your_panel_app) + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + +app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py new file mode 100644 index 0000000000000..8e20350dd036c --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py @@ -0,0 +1,50 @@ +# app.py +import datetime as dt + +import panel as pn +from panel_frontend import PanelFrontend +from panel_utils import AppStateWatcher + +import lightning as L + +pn.extension(sizing_mode="stretch_width") + +def your_panel_app(app: AppStateWatcher): + + @pn.depends(app.param.state) + def last_update(_): + return f'last_update: {app.state.last_update}' + + return pn.Column( + last_update, + ) + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + + self._frontend = PanelFrontend(render_fn=your_panel_app) + self._last_update=dt.datetime.now() + self.last_update=self._last_update.isoformat() + + def run(self): + now = dt.datetime.now() + if (now-self._last_update).microseconds>200: + self._last_update=now + self.last_update=self._last_update.isoformat() + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def run(self) -> None: + self.lit_panel.run() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + +app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py new file mode 100644 index 0000000000000..651a10b118a75 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py @@ -0,0 +1,43 @@ +# app.py +import lightning as L +import panel as pn +from panel_frontend import PanelFrontend +from panel_utils import AppStateWatcher + +pn.extension(sizing_mode="stretch_width") + +def your_panel_app(app: AppStateWatcher): + + submit_button = pn.widgets.Button(name="submit") + + @pn.depends(submit_button, watch=True) + def submit(_): + app.state.count += 1 + + @pn.depends(app.param.state) + def current_count(_): + return f'current count: {app.state.count}' + + return pn.Column( + submit_button, + current_count, + ) + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend(render_fn=your_panel_app) + self.count=0 + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + +app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py b/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py deleted file mode 100644 index 0d2bf61c9c5a0..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/app_pull_state.py +++ /dev/null @@ -1,72 +0,0 @@ -"""This app demonstrates how the PanelFrontend can read/ pull the global app state and display it.""" -import datetime -import json -from pathlib import Path - -import panel as pn -from panel_frontend import PanelFrontend - -import lightning as L - -DATETIME_FORMAT = "%Y-%m-%d, %H:%M:%S.%f" - -APP_STATE = Path(__file__).parent / "app_state.json" - - -def read_app_state(): - with open(APP_STATE) as json_file: - return json.load(json_file) - - -def save_app_state(state): - with open(APP_STATE, "w") as outfile: - json.dump(state, outfile) - - -def to_string(value: datetime.datetime) -> str: - return value.strftime(DATETIME_FORMAT) - - -def render_fn(self): - global_app_state_pane = pn.pane.JSON(depth=2) - last_local_update_pane = pn.pane.Str() - - def update(): - last_local_update_pane.object = "last local update: " + to_string(datetime.datetime.now()) - - # Todo: Figure out the right way to read/ pull the app state - global_app_state_pane.object = read_app_state() - - # Todo: Figure out how to schedule the callback globally to make the app scale - # There is no reason that each session should read or pull into memory individually - pn.state.add_periodic_callback(update, period=1000) - # Todo: Refactor the Panel app implementation to a more reactive api - # Todo: Giver the Panel app a nicer UX - return pn.Column(last_local_update_pane, "## Global App State", global_app_state_pane) - - -class Flow(L.LightningFlow): - def __init__(self): - super().__init__() - - self.panel_frontend = PanelFrontend(render_fn=render_fn) - self._last_global_update = datetime.datetime.now() - self.last_global_update = to_string(self._last_global_update) - - def run(self): - self.panel_frontend.run() - now = datetime.datetime.now() - if (now - self._last_global_update).microseconds >= 100: - save_app_state(self.state) - self._last_global_update = now - self.last_global_update = to_string(now) - - def configure_layout(self): - tab1 = {"name": "Home", "content": self.panel_frontend} - return tab1 - - -app = L.LightningApp(Flow()) - -if __name__.startswith("bokeh"): - render_fn(None).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py b/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py index da56cdfbb0a28..b0eb194060e14 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py @@ -1,22 +1,27 @@ -# app.py -import lightning as L -from lightning.app.frontend.stream_lit import StreamlitFrontend -import streamlit as st - -def your_streamlit_app(lightning_app_state): - st.write('hello world') - -class LitStreamlit(L.LightningFlow): - def configure_layout(self): - return StreamlitFrontend(render_fn=your_streamlit_app) - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - +# app.py +from typing import Union +import lightning as L +from lightning.app.frontend.stream_lit import StreamlitFrontend +import streamlit as st +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState +import os + +def your_streamlit_app(lightning_app_state): + st.write('hello world') + st.write(lightning_app_state) + +class LitStreamlit(L.LightningFlow): + def configure_layout(self): + return StreamlitFrontend(render_fn=your_streamlit_app) + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_streamlit = LitStreamlit() + + def configure_layout(self): + tab1 = {"name": "home", "content": self.lit_streamlit} + return tab1 + app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py index 43c2ccb3b3489..1d4a873756d72 100644 --- a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py @@ -11,12 +11,6 @@ import pathlib logger = logging.getLogger("PanelFrontend") -logger.setLevel(logging.DEBUG) - -handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter("%(asctime)s - %(message)s") -handler.setFormatter(formatter) -logger.addHandler(handler) class PanelFrontend(Frontend): @@ -30,10 +24,10 @@ def __init__(self, render_fn: Callable): # Would like to accept a `render_file` ) self.render_fn = render_fn - logger.info("init finished") + logger.debug("initialized") def start_server(self, host: str, port: int) -> None: - logger.info("starting server %s %s", host, port) + logger.debug("starting server %s %s", host, port) env = os.environ.copy() env["LIGHTNING_FLOW_NAME"] = self.flow.name env["LIGHTNING_RENDER_FUNCTION"] = self.render_fn.__name__ @@ -46,7 +40,7 @@ def start_server(self, host: str, port: int) -> None: self._process = subprocess.Popen( [ sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn.py", + pathlib.Path(__file__).parent / "panel_serve.py", ], env=env, # stdout=stdout, diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_plugin.py b/docs/source-app/workflows/add_web_ui/panel/panel_plugin.py deleted file mode 100644 index 10034c9d2c993..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/panel_plugin.py +++ /dev/null @@ -1,65 +0,0 @@ -import asyncio -import logging -import os -import sys -import threading -import time - -import param -import websockets - -from lightning_app.core.constants import APP_SERVER_PORT -from lightning_app.utilities.app_helpers import AppStateType, BaseStatePlugin - -logger = logging.getLogger("PanelPlugin") -logger.setLevel(logging.DEBUG) - -handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter('%(asctime)s - %(message)s') -handler.setFormatter(formatter) -logger.addHandler(handler) - -class ParamState(param.Parameterized): - value: int = param.Integer() - -def target_fn(param_state: ParamState): - async def update_fn(param_state: ParamState): - url = "localhost:8080" if "LIGHTNING_APP_STATE_URL" in os.environ else f"localhost:{APP_SERVER_PORT}" - ws_url = f"ws://{url}/api/v1/ws" - last_updated = time.time() - logger.info("connecting to web sockets %s", ws_url) - async with websockets.connect(ws_url) as websocket: - while True: - await websocket.recv() - while (time.time() - last_updated) < 1: - time.sleep(0.1) - last_updated = time.time() - param_state.value += 1 - logger.info("App state changed") - - asyncio.run(update_fn(param_state)) - - - -class PanelStatePlugin(BaseStatePlugin): - def __init__(self): - super().__init__() - import panel as pn - self.param_state = ParamState(value=1) - - if "_lightning_websocket_thread" not in pn.state.cache: - logger.info("starting thread") - thread = threading.Thread(target=target_fn, args=(self.param_state, )) - pn.state.cache["_lightning_websocket_thread"] = thread - thread.setDaemon(True) - thread.start() - logger.info("thread started") - - def should_update_app(self, deep_diff): - return deep_diff - - def get_context(self): - return {"type": AppStateType.DEFAULT.value} - - def render_non_authorized(self): - pass diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_serve.py b/docs/source-app/workflows/add_web_ui/panel/panel_serve.py new file mode 100644 index 0000000000000..30ef77b7ea0bc --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/panel_serve.py @@ -0,0 +1,45 @@ +"""This file gets run by streamlit, which we launch within Lightning. + +From here, we will call the render function that the user provided in ``configure_layout``. +""" +from __future__ import annotations +import logging +import os +import sys + +import panel as pn + +from lightning_app.frontend.streamlit_base import _get_render_fn_from_environment +from panel_utils import AppStateWatcher + +logger = logging.getLogger("PanelFrontend") + +logger.setLevel(logging.DEBUG) + +logger.debug("starting plugin") +logger.debug("plugin started") + +app_state_watcher: None | AppStateWatcher = None + +def main(): + def view(): + global app_state_watcher + if not app_state_watcher: + app_state_watcher = AppStateWatcher() + render_fn = _get_render_fn_from_environment() + return render_fn(app_state_watcher) + + logger.debug("Panel server starting") + port=int(os.environ["LIGHTNING_RENDER_PORT"]) + address=os.environ["LIGHTNING_RENDER_ADDRESS"] + url = os.environ["LIGHTNING_FLOW_NAME"] + pn.serve({url: view}, address=address, port=port, websocket_origin="*", show=False) + logger.debug("Panel server started on port %s:%s/%s", address, port, url) + +# os.environ['LIGHTNING_FLOW_NAME']= 'root.lit_panel' +# os.environ['LIGHTNING_RENDER_FUNCTION']= 'your_panel_app' +# os.environ['LIGHTNING_RENDER_MODULE_FILE']= 'C:\\repos\\private\\lightning\\docs\\source-app\\workflows\\add_web_ui\\panel\\app.py' +# os.environ['LIGHTNING_RENDER_ADDRESS']= 'localhost' +# os.environ['LIGHTNING_RENDER_PORT']= '61965' + +main() \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py deleted file mode 100644 index c4794ff8748b5..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/panel_serve_render_fn.py +++ /dev/null @@ -1,66 +0,0 @@ -"""This file gets run by streamlit, which we launch within Lightning. - -From here, we will call the render function that the user provided in ``configure_layout``. -""" -import logging -import os -import pydoc -import sys -from typing import Callable, Union - -import panel as pn - -from lightning_app.core.flow import LightningFlow -from lightning_app.utilities.state import AppState -from panel_plugin import PanelStatePlugin - -logger = logging.getLogger("PanelFrontend") -logger.setLevel(logging.DEBUG) - -handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter('%(asctime)s - %(message)s') -handler.setFormatter(formatter) -logger.addHandler(handler) - -logger.info("starting plugin") -app_state = AppState(plugin=PanelStatePlugin()) # -logger.info("plugin started") - -def _get_render_fn_from_environment() -> Callable: - render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] - render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] - module = pydoc.importfile(render_fn_module_file) - return getattr(module, render_fn_name) - - -def _app_state_to_flow_scope(state: AppState, flow: Union[str, LightningFlow]) -> AppState: - """Returns a new AppState with the scope reduced to the given flow, as if the given flow as the root.""" - flow_name = flow.name if isinstance(flow, LightningFlow) else flow - flow_name_parts = flow_name.split(".")[1:] # exclude root - flow_state = state - for part in flow_name_parts: - flow_state = getattr(flow_state, part) - return flow_state - - -def view(): - flow_state = _app_state_to_flow_scope(app_state, flow=os.environ["LIGHTNING_FLOW_NAME"]) - render_fn = _get_render_fn_from_environment() - - return render_fn(app_state) - -def main(): - logger.info("Panel server starting") - port=int(os.environ["LIGHTNING_RENDER_PORT"]) - address=os.environ["LIGHTNING_RENDER_ADDRESS"] - url = os.environ["LIGHTNING_FLOW_NAME"] - pn.serve({url: view}, address=address, port=port, websocket_origin="*", show=False) - logger.info("Panel server started on port %s:%s/%s", address, port, url) - -# os.environ['LIGHTNING_FLOW_NAME']= 'root.lit_panel' -# os.environ['LIGHTNING_RENDER_FUNCTION']= 'your_panel_app' -# os.environ['LIGHTNING_RENDER_MODULE_FILE']= 'C:\\repos\\private\\lightning\\docs\\source-app\\workflows\\add_web_ui\\panel\\app.py' -# os.environ['LIGHTNING_RENDER_ADDRESS']= 'localhost' -# os.environ['LIGHTNING_RENDER_PORT']= '61965' - -main() \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_utils.py b/docs/source-app/workflows/add_web_ui/panel/panel_utils.py new file mode 100644 index 0000000000000..c5ec6953e9c9f --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/panel_utils.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from typing import Callable + +import param +import websockets + +from lightning_app.core.constants import APP_SERVER_PORT +from lightning_app.frontend.streamlit_base import _app_state_to_flow_scope +from lightning_app.utilities.state import AppState + +logger = logging.getLogger("PanelFrontend") + +_CALLBACKS = [] +_THREAD: None | threading.Thread = None + +def _target_fn(): + async def update_fn(): + url = "localhost:8080" if "LIGHTNING_APP_STATE_URL" in os.environ else f"localhost:{APP_SERVER_PORT}" + ws_url = f"ws://{url}/api/v1/ws" + logger.debug("connecting to web socket %s", ws_url) + async with websockets.connect(ws_url) as websocket: + while True: + await websocket.recv() + # while (time.time() - last_updated) < 0.2: + # time.sleep(0.05) + logger.debug("App State Changed. Running callbacks") + for callback in _CALLBACKS: + callback() + + + asyncio.run(update_fn()) + +def _start_websocket(): + global _THREAD + if not _THREAD: + logger.debug("starting thread") + _THREAD = threading.Thread(target=_target_fn) + _THREAD.setDaemon(True) + _THREAD.start() + logger.debug("thread started") + +def watch_app_state(callback: Callable): + _CALLBACKS.append(callback) + + _start_websocket() + +def get_flow_state(): + app_state = AppState() + app_state._request_state() + flow = os.environ["LIGHTNING_FLOW_NAME"] + flow_state = _app_state_to_flow_scope(app_state, flow) + return flow_state + +class AppStateWatcher(param.Parameterized): + state: AppState = param.ClassSelector(class_=AppState) + + def __init__(self): + app_state = self._get_flow_state() + super().__init__(state=app_state) + watch_app_state(self.handle_state_changed) + + def _get_flow_state(self): + return get_flow_state() + + def _request_state(self): + self.state = self._get_flow_state() + logger.debug("Request app state") + + def handle_state_changed(self): + logger.debug("Handle app state changed") + self._request_state() diff --git a/docs/source-app/workflows/add_web_ui/panel/test_panel.py b/docs/source-app/workflows/add_web_ui/panel/test_panel.py index 2dc9717c19971..bc8f766796ccb 100644 --- a/docs/source-app/workflows/add_web_ui/panel/test_panel.py +++ b/docs/source-app/workflows/add_web_ui/panel/test_panel.py @@ -1,16 +1,16 @@ -import logging -import os -import pydoc -import sys -from typing import Callable, Union - -import panel as pn - -from lightning_app.core.flow import LightningFlow -from lightning_app.utilities.state import AppState -from panel_plugin import PanelStatePlugin - - -def test_param_state(): - app_state = AppState(plugin=PanelStatePlugin()) # - assert hasattr(app_state._plugin.param_state) +import logging +import os +import pydoc +import sys +from typing import Callable, Union + +import panel as pn + +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState +from panel_plugin import PanelStatePlugin + + +def test_param_state(): + app_state = AppState(plugin=PanelStatePlugin()) # + assert hasattr(app_state._plugin.param_state) From b4dc86da5c6e85c95d539f297eb9a76a3ccf3175 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 8 Jul 2022 06:02:52 +0000 Subject: [PATCH 005/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../workflows/add_web_ui/panel/app_basic.py | 11 ++++++--- .../panel/app_interact_from_component.py | 23 +++++++++++-------- .../panel/app_interact_from_frontend.py | 21 ++++++++++------- .../add_web_ui/panel/app_streamlit.py | 14 +++++++---- .../add_web_ui/panel/panel_frontend.py | 8 +++---- .../workflows/add_web_ui/panel/panel_serve.py | 13 +++++++---- .../workflows/add_web_ui/panel/panel_utils.py | 10 +++++--- .../workflows/add_web_ui/panel/test_panel.py | 6 ++--- 8 files changed, 66 insertions(+), 40 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/app_basic.py index 790ff46e66dcd..d96bdbd0c893a 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_basic.py @@ -1,20 +1,24 @@ # app.py -import lightning as L import panel as pn from panel_frontend import PanelFrontend from panel_utils import AppStateWatcher +import lightning as L + + def your_panel_app(app: AppStateWatcher): return pn.pane.Markdown("hello") + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() self._frontend = PanelFrontend(render_fn=your_panel_app) - + def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -23,4 +27,5 @@ def __init__(self): def configure_layout(self): return {"name": "home", "content": self.lit_panel} -app = L.LightningApp(LitApp()) \ No newline at end of file + +app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py index 8e20350dd036c..30532a54eb517 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py @@ -9,33 +9,35 @@ pn.extension(sizing_mode="stretch_width") + def your_panel_app(app: AppStateWatcher): - @pn.depends(app.param.state) def last_update(_): - return f'last_update: {app.state.last_update}' + return f"last_update: {app.state.last_update}" return pn.Column( last_update, ) + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - + self._frontend = PanelFrontend(render_fn=your_panel_app) - self._last_update=dt.datetime.now() - self.last_update=self._last_update.isoformat() + self._last_update = dt.datetime.now() + self.last_update = self._last_update.isoformat() def run(self): now = dt.datetime.now() - if (now-self._last_update).microseconds>200: - self._last_update=now - self.last_update=self._last_update.isoformat() - + if (now - self._last_update).microseconds > 200: + self._last_update = now + self.last_update = self._last_update.isoformat() + def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -47,4 +49,5 @@ def run(self) -> None: def configure_layout(self): return {"name": "home", "content": self.lit_panel} -app = L.LightningApp(LitApp()) \ No newline at end of file + +app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py index 651a10b118a75..143244e9fde43 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py @@ -1,37 +1,41 @@ # app.py -import lightning as L import panel as pn from panel_frontend import PanelFrontend from panel_utils import AppStateWatcher +import lightning as L + pn.extension(sizing_mode="stretch_width") + def your_panel_app(app: AppStateWatcher): - + submit_button = pn.widgets.Button(name="submit") - + @pn.depends(submit_button, watch=True) def submit(_): - app.state.count += 1 + app.state.count += 1 @pn.depends(app.param.state) def current_count(_): - return f'current count: {app.state.count}' + return f"current count: {app.state.count}" return pn.Column( submit_button, current_count, ) + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() self._frontend = PanelFrontend(render_fn=your_panel_app) - self.count=0 - + self.count = 0 + def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -40,4 +44,5 @@ def __init__(self): def configure_layout(self): return {"name": "home", "content": self.lit_panel} -app = L.LightningApp(LitApp()) \ No newline at end of file + +app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py b/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py index b0eb194060e14..74fe4764c7fae 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py +++ b/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py @@ -1,20 +1,25 @@ # app.py +import os from typing import Union + +import streamlit as st + import lightning as L from lightning.app.frontend.stream_lit import StreamlitFrontend -import streamlit as st from lightning_app.core.flow import LightningFlow from lightning_app.utilities.state import AppState -import os + def your_streamlit_app(lightning_app_state): - st.write('hello world') + st.write("hello world") st.write(lightning_app_state) + class LitStreamlit(L.LightningFlow): def configure_layout(self): return StreamlitFrontend(render_fn=your_streamlit_app) + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -24,4 +29,5 @@ def configure_layout(self): tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 -app = L.LightningApp(LitApp()) \ No newline at end of file + +app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py index 1d4a873756d72..5dca948af37a3 100644 --- a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py @@ -1,6 +1,7 @@ import inspect import logging import os +import pathlib import subprocess import sys from typing import Callable @@ -8,14 +9,13 @@ from lightning_app.frontend.frontend import Frontend from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile -import pathlib logger = logging.getLogger("PanelFrontend") + class PanelFrontend(Frontend): - @requires("panel") - def __init__(self, render_fn: Callable): # Would like to accept a `render_file` arguemnt too in the future + def __init__(self, render_fn: Callable): # Would like to accept a `render_file` arguemnt too in the future super().__init__() if inspect.ismethod(render_fn): @@ -33,7 +33,7 @@ def start_server(self, host: str, port: int) -> None: env["LIGHTNING_RENDER_FUNCTION"] = self.render_fn.__name__ env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(self.render_fn).__file__ env["LIGHTNING_RENDER_PORT"] = str(port) - env["LIGHTNING_RENDER_ADDRESS"] = str(host) + env["LIGHTNING_RENDER_ADDRESS"] = str(host) std_err_out = get_frontend_logfile("error.log") std_out_out = get_frontend_logfile("output.log") with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_serve.py b/docs/source-app/workflows/add_web_ui/panel/panel_serve.py index 30ef77b7ea0bc..648ff25c8f55c 100644 --- a/docs/source-app/workflows/add_web_ui/panel/panel_serve.py +++ b/docs/source-app/workflows/add_web_ui/panel/panel_serve.py @@ -3,14 +3,15 @@ From here, we will call the render function that the user provided in ``configure_layout``. """ from __future__ import annotations + import logging import os import sys import panel as pn +from panel_utils import AppStateWatcher from lightning_app.frontend.streamlit_base import _get_render_fn_from_environment -from panel_utils import AppStateWatcher logger = logging.getLogger("PanelFrontend") @@ -21,6 +22,7 @@ app_state_watcher: None | AppStateWatcher = None + def main(): def view(): global app_state_watcher @@ -28,18 +30,19 @@ def view(): app_state_watcher = AppStateWatcher() render_fn = _get_render_fn_from_environment() return render_fn(app_state_watcher) - + logger.debug("Panel server starting") - port=int(os.environ["LIGHTNING_RENDER_PORT"]) - address=os.environ["LIGHTNING_RENDER_ADDRESS"] + port = int(os.environ["LIGHTNING_RENDER_PORT"]) + address = os.environ["LIGHTNING_RENDER_ADDRESS"] url = os.environ["LIGHTNING_FLOW_NAME"] pn.serve({url: view}, address=address, port=port, websocket_origin="*", show=False) logger.debug("Panel server started on port %s:%s/%s", address, port, url) + # os.environ['LIGHTNING_FLOW_NAME']= 'root.lit_panel' # os.environ['LIGHTNING_RENDER_FUNCTION']= 'your_panel_app' # os.environ['LIGHTNING_RENDER_MODULE_FILE']= 'C:\\repos\\private\\lightning\\docs\\source-app\\workflows\\add_web_ui\\panel\\app.py' # os.environ['LIGHTNING_RENDER_ADDRESS']= 'localhost' # os.environ['LIGHTNING_RENDER_PORT']= '61965' -main() \ No newline at end of file +main() diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_utils.py b/docs/source-app/workflows/add_web_ui/panel/panel_utils.py index c5ec6953e9c9f..3ab5b095cdba2 100644 --- a/docs/source-app/workflows/add_web_ui/panel/panel_utils.py +++ b/docs/source-app/workflows/add_web_ui/panel/panel_utils.py @@ -18,6 +18,7 @@ _CALLBACKS = [] _THREAD: None | threading.Thread = None + def _target_fn(): async def update_fn(): url = "localhost:8080" if "LIGHTNING_APP_STATE_URL" in os.environ else f"localhost:{APP_SERVER_PORT}" @@ -31,10 +32,10 @@ async def update_fn(): logger.debug("App State Changed. Running callbacks") for callback in _CALLBACKS: callback() - asyncio.run(update_fn()) + def _start_websocket(): global _THREAD if not _THREAD: @@ -44,11 +45,13 @@ def _start_websocket(): _THREAD.start() logger.debug("thread started") + def watch_app_state(callback: Callable): _CALLBACKS.append(callback) - + _start_websocket() + def get_flow_state(): app_state = AppState() app_state._request_state() @@ -56,9 +59,10 @@ def get_flow_state(): flow_state = _app_state_to_flow_scope(app_state, flow) return flow_state + class AppStateWatcher(param.Parameterized): state: AppState = param.ClassSelector(class_=AppState) - + def __init__(self): app_state = self._get_flow_state() super().__init__(state=app_state) diff --git a/docs/source-app/workflows/add_web_ui/panel/test_panel.py b/docs/source-app/workflows/add_web_ui/panel/test_panel.py index bc8f766796ccb..f65708a6bc1d6 100644 --- a/docs/source-app/workflows/add_web_ui/panel/test_panel.py +++ b/docs/source-app/workflows/add_web_ui/panel/test_panel.py @@ -5,12 +5,12 @@ from typing import Callable, Union import panel as pn +from panel_plugin import PanelStatePlugin from lightning_app.core.flow import LightningFlow from lightning_app.utilities.state import AppState -from panel_plugin import PanelStatePlugin def test_param_state(): - app_state = AppState(plugin=PanelStatePlugin()) # - assert hasattr(app_state._plugin.param_state) + app_state = AppState(plugin=PanelStatePlugin()) # + assert hasattr(app_state._plugin.param_state) From 8e50ca1edc114b7517445b4ce4f94a4e1e5faada Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sat, 9 Jul 2022 12:56:47 +0200 Subject: [PATCH 006/103] refactor --- .gitignore | 3 + .../panel/{ => examples}/app_basic.py | 49 +- .../app_interact_from_component.py | 98 ++-- .../app_interact_from_frontend.py | 84 ++-- .../panel/{ => examples}/app_streamlit.py | 53 +- .../add_web_ui/panel/panel_frontend.py | 53 -- .../workflows/add_web_ui/panel/panel_serve.py | 45 -- .../workflows/add_web_ui/panel/panel_utils.py | 76 --- .../workflows/add_web_ui/panel/test_panel.py | 16 - src/lightning_app/frontend/panel/__init__.py | 5 + .../frontend/panel/panel_frontend.py | 83 ++++ .../frontend/panel/panel_serve_render_fn.py | 58 +++ .../frontend/utilities/__init__.py | 0 .../frontend/utilities/app_state_comm.py | 83 ++++ .../frontend/utilities/app_state_watcher.py | 83 ++++ src/lightning_app/frontend/utilities/other.py | 64 +++ src/lightning_app/utilities/state.py | 462 +++++++++--------- tests/tests_app/frontend/panel/__init__.py | 0 .../frontend/panel/test_panel_frontend.py | 94 ++++ .../panel/test_panel_serve_render_fn.py | 59 +++ .../tests_app/frontend/utilities/__init__.py | 0 .../tests_app/frontend/utilities/conftest.py | 81 +++ .../frontend/utilities/test_app_state_comm.py | 40 ++ .../utilities/test_app_state_watcher.py | 36 ++ .../frontend/utilities/test_other.py | 46 ++ 25 files changed, 1108 insertions(+), 563 deletions(-) rename docs/source-app/workflows/add_web_ui/panel/{ => examples}/app_basic.py (78%) rename docs/source-app/workflows/add_web_ui/panel/{ => examples}/app_interact_from_component.py (93%) rename docs/source-app/workflows/add_web_ui/panel/{ => examples}/app_interact_from_frontend.py (92%) rename docs/source-app/workflows/add_web_ui/panel/{ => examples}/app_streamlit.py (94%) delete mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_frontend.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_serve.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/panel_utils.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/test_panel.py create mode 100644 src/lightning_app/frontend/panel/__init__.py create mode 100644 src/lightning_app/frontend/panel/panel_frontend.py create mode 100644 src/lightning_app/frontend/panel/panel_serve_render_fn.py create mode 100644 src/lightning_app/frontend/utilities/__init__.py create mode 100644 src/lightning_app/frontend/utilities/app_state_comm.py create mode 100644 src/lightning_app/frontend/utilities/app_state_watcher.py create mode 100644 src/lightning_app/frontend/utilities/other.py create mode 100644 tests/tests_app/frontend/panel/__init__.py create mode 100644 tests/tests_app/frontend/panel/test_panel_frontend.py create mode 100644 tests/tests_app/frontend/panel/test_panel_serve_render_fn.py create mode 100644 tests/tests_app/frontend/utilities/__init__.py create mode 100644 tests/tests_app/frontend/utilities/conftest.py create mode 100644 tests/tests_app/frontend/utilities/test_app_state_comm.py create mode 100644 tests/tests_app/frontend/utilities/test_app_state_watcher.py create mode 100644 tests/tests_app/frontend/utilities/test_other.py diff --git a/.gitignore b/.gitignore index 47b9bfff92523..71ad77140b434 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cifar-10-batches-py # ctags tags .tags + +# Lightning StreamlitFrontend or PanelFrontend +.storage \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py similarity index 78% rename from docs/source-app/workflows/add_web_ui/panel/app_basic.py rename to docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py index 790ff46e66dcd..1359bd586d877 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py @@ -1,26 +1,25 @@ -# app.py -import lightning as L -import panel as pn -from panel_frontend import PanelFrontend -from panel_utils import AppStateWatcher - -def your_panel_app(app: AppStateWatcher): - return pn.pane.Markdown("hello") - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend(render_fn=your_panel_app) - - def configure_layout(self): - return self._frontend - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - +# app.py +import panel as pn +from lightning_app.frontend.panel import PanelFrontend +import lightning as L + +def your_panel_app(app): + return pn.pane.Markdown("hello") + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend(render_fn=your_panel_app) + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py similarity index 93% rename from docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py rename to docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py index 8e20350dd036c..30b45fffd542e 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_component.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py @@ -1,50 +1,50 @@ -# app.py -import datetime as dt - -import panel as pn -from panel_frontend import PanelFrontend -from panel_utils import AppStateWatcher - -import lightning as L - -pn.extension(sizing_mode="stretch_width") - -def your_panel_app(app: AppStateWatcher): - - @pn.depends(app.param.state) - def last_update(_): - return f'last_update: {app.state.last_update}' - - return pn.Column( - last_update, - ) - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - - self._frontend = PanelFrontend(render_fn=your_panel_app) - self._last_update=dt.datetime.now() - self.last_update=self._last_update.isoformat() - - def run(self): - now = dt.datetime.now() - if (now-self._last_update).microseconds>200: - self._last_update=now - self.last_update=self._last_update.isoformat() - - def configure_layout(self): - return self._frontend - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self) -> None: - self.lit_panel.run() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - +# app.py +import datetime as dt + +import panel as pn +from panel_frontend import PanelFrontend +from app_state_watcher import AppStateWatcher + +import lightning as L + +pn.extension(sizing_mode="stretch_width") + +def your_panel_app(app: AppStateWatcher): + + @pn.depends(app.param.state) + def last_update(_): + return f'last_update: {app.state.last_update}' + + return pn.Column( + last_update, + ) + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + + self._frontend = PanelFrontend(render_fn=your_panel_app) + self._last_update=dt.datetime.now() + self.last_update=self._last_update.isoformat() + + def run(self): + now = dt.datetime.now() + if (now-self._last_update).microseconds>200: + self._last_update=now + self.last_update=self._last_update.isoformat() + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def run(self) -> None: + self.lit_panel.run() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py similarity index 92% rename from docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py rename to docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py index 651a10b118a75..2c9f2007ffb89 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_interact_from_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py @@ -1,43 +1,43 @@ -# app.py -import lightning as L -import panel as pn -from panel_frontend import PanelFrontend -from panel_utils import AppStateWatcher - -pn.extension(sizing_mode="stretch_width") - -def your_panel_app(app: AppStateWatcher): - - submit_button = pn.widgets.Button(name="submit") - - @pn.depends(submit_button, watch=True) - def submit(_): - app.state.count += 1 - - @pn.depends(app.param.state) - def current_count(_): - return f'current count: {app.state.count}' - - return pn.Column( - submit_button, - current_count, - ) - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend(render_fn=your_panel_app) - self.count=0 - - def configure_layout(self): - return self._frontend - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - +# app.py +import lightning as L +import panel as pn +from panel_frontend import PanelFrontend +from app_state_watcher import AppStateWatcher + +pn.extension(sizing_mode="stretch_width") + +def your_panel_app(app: AppStateWatcher): + + submit_button = pn.widgets.Button(name="submit") + + @pn.depends(submit_button, watch=True) + def submit(_): + app.state.count += 1 + + @pn.depends(app.param.state) + def current_count(_): + return f'current count: {app.state.count}' + + return pn.Column( + submit_button, + current_count, + ) + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend(render_fn=your_panel_app) + self.count=0 + + def configure_layout(self): + return self._frontend + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py similarity index 94% rename from docs/source-app/workflows/add_web_ui/panel/app_streamlit.py rename to docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py index b0eb194060e14..7f41f4182576d 100644 --- a/docs/source-app/workflows/add_web_ui/panel/app_streamlit.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py @@ -1,27 +1,28 @@ -# app.py -from typing import Union -import lightning as L -from lightning.app.frontend.stream_lit import StreamlitFrontend -import streamlit as st -from lightning_app.core.flow import LightningFlow -from lightning_app.utilities.state import AppState -import os - -def your_streamlit_app(lightning_app_state): - st.write('hello world') - st.write(lightning_app_state) - -class LitStreamlit(L.LightningFlow): - def configure_layout(self): - return StreamlitFrontend(render_fn=your_streamlit_app) - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - +# app.py +from typing import Union +import lightning as L +from lightning.app.frontend.stream_lit import StreamlitFrontend +import streamlit as st +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState +import os + +def your_streamlit_app(lightning_app_state): + st.write('hello world') + st.write(lightning_app_state) + st.write(os.environ["LIGHTNING_FLOW_NAME"]) + +class LitStreamlit(L.LightningFlow): + def configure_layout(self): + return StreamlitFrontend(render_fn=your_streamlit_app) + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_streamlit = LitStreamlit() + + def configure_layout(self): + tab1 = {"name": "home", "content": self.lit_streamlit} + return tab1 + app = L.LightningApp(LitApp()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py deleted file mode 100644 index 1d4a873756d72..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/panel_frontend.py +++ /dev/null @@ -1,53 +0,0 @@ -import inspect -import logging -import os -import subprocess -import sys -from typing import Callable - -from lightning_app.frontend.frontend import Frontend -from lightning_app.utilities.imports import requires -from lightning_app.utilities.log import get_frontend_logfile -import pathlib - -logger = logging.getLogger("PanelFrontend") - -class PanelFrontend(Frontend): - - @requires("panel") - def __init__(self, render_fn: Callable): # Would like to accept a `render_file` arguemnt too in the future - super().__init__() - - if inspect.ismethod(render_fn): - raise TypeError( - "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a pure function." - ) - - self.render_fn = render_fn - logger.debug("initialized") - - def start_server(self, host: str, port: int) -> None: - logger.debug("starting server %s %s", host, port) - env = os.environ.copy() - env["LIGHTNING_FLOW_NAME"] = self.flow.name - env["LIGHTNING_RENDER_FUNCTION"] = self.render_fn.__name__ - env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(self.render_fn).__file__ - env["LIGHTNING_RENDER_PORT"] = str(port) - env["LIGHTNING_RENDER_ADDRESS"] = str(host) - std_err_out = get_frontend_logfile("error.log") - std_out_out = get_frontend_logfile("output.log") - with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: - self._process = subprocess.Popen( - [ - sys.executable, - pathlib.Path(__file__).parent / "panel_serve.py", - ], - env=env, - # stdout=stdout, - # stderr=stderr, - ) - - def stop_server(self) -> None: - if self._process is None: - raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") - self._process.kill() diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_serve.py b/docs/source-app/workflows/add_web_ui/panel/panel_serve.py deleted file mode 100644 index 30ef77b7ea0bc..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/panel_serve.py +++ /dev/null @@ -1,45 +0,0 @@ -"""This file gets run by streamlit, which we launch within Lightning. - -From here, we will call the render function that the user provided in ``configure_layout``. -""" -from __future__ import annotations -import logging -import os -import sys - -import panel as pn - -from lightning_app.frontend.streamlit_base import _get_render_fn_from_environment -from panel_utils import AppStateWatcher - -logger = logging.getLogger("PanelFrontend") - -logger.setLevel(logging.DEBUG) - -logger.debug("starting plugin") -logger.debug("plugin started") - -app_state_watcher: None | AppStateWatcher = None - -def main(): - def view(): - global app_state_watcher - if not app_state_watcher: - app_state_watcher = AppStateWatcher() - render_fn = _get_render_fn_from_environment() - return render_fn(app_state_watcher) - - logger.debug("Panel server starting") - port=int(os.environ["LIGHTNING_RENDER_PORT"]) - address=os.environ["LIGHTNING_RENDER_ADDRESS"] - url = os.environ["LIGHTNING_FLOW_NAME"] - pn.serve({url: view}, address=address, port=port, websocket_origin="*", show=False) - logger.debug("Panel server started on port %s:%s/%s", address, port, url) - -# os.environ['LIGHTNING_FLOW_NAME']= 'root.lit_panel' -# os.environ['LIGHTNING_RENDER_FUNCTION']= 'your_panel_app' -# os.environ['LIGHTNING_RENDER_MODULE_FILE']= 'C:\\repos\\private\\lightning\\docs\\source-app\\workflows\\add_web_ui\\panel\\app.py' -# os.environ['LIGHTNING_RENDER_ADDRESS']= 'localhost' -# os.environ['LIGHTNING_RENDER_PORT']= '61965' - -main() \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/panel_utils.py b/docs/source-app/workflows/add_web_ui/panel/panel_utils.py deleted file mode 100644 index c5ec6953e9c9f..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/panel_utils.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import os -import threading -from typing import Callable - -import param -import websockets - -from lightning_app.core.constants import APP_SERVER_PORT -from lightning_app.frontend.streamlit_base import _app_state_to_flow_scope -from lightning_app.utilities.state import AppState - -logger = logging.getLogger("PanelFrontend") - -_CALLBACKS = [] -_THREAD: None | threading.Thread = None - -def _target_fn(): - async def update_fn(): - url = "localhost:8080" if "LIGHTNING_APP_STATE_URL" in os.environ else f"localhost:{APP_SERVER_PORT}" - ws_url = f"ws://{url}/api/v1/ws" - logger.debug("connecting to web socket %s", ws_url) - async with websockets.connect(ws_url) as websocket: - while True: - await websocket.recv() - # while (time.time() - last_updated) < 0.2: - # time.sleep(0.05) - logger.debug("App State Changed. Running callbacks") - for callback in _CALLBACKS: - callback() - - - asyncio.run(update_fn()) - -def _start_websocket(): - global _THREAD - if not _THREAD: - logger.debug("starting thread") - _THREAD = threading.Thread(target=_target_fn) - _THREAD.setDaemon(True) - _THREAD.start() - logger.debug("thread started") - -def watch_app_state(callback: Callable): - _CALLBACKS.append(callback) - - _start_websocket() - -def get_flow_state(): - app_state = AppState() - app_state._request_state() - flow = os.environ["LIGHTNING_FLOW_NAME"] - flow_state = _app_state_to_flow_scope(app_state, flow) - return flow_state - -class AppStateWatcher(param.Parameterized): - state: AppState = param.ClassSelector(class_=AppState) - - def __init__(self): - app_state = self._get_flow_state() - super().__init__(state=app_state) - watch_app_state(self.handle_state_changed) - - def _get_flow_state(self): - return get_flow_state() - - def _request_state(self): - self.state = self._get_flow_state() - logger.debug("Request app state") - - def handle_state_changed(self): - logger.debug("Handle app state changed") - self._request_state() diff --git a/docs/source-app/workflows/add_web_ui/panel/test_panel.py b/docs/source-app/workflows/add_web_ui/panel/test_panel.py deleted file mode 100644 index bc8f766796ccb..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/test_panel.py +++ /dev/null @@ -1,16 +0,0 @@ -import logging -import os -import pydoc -import sys -from typing import Callable, Union - -import panel as pn - -from lightning_app.core.flow import LightningFlow -from lightning_app.utilities.state import AppState -from panel_plugin import PanelStatePlugin - - -def test_param_state(): - app_state = AppState(plugin=PanelStatePlugin()) # - assert hasattr(app_state._plugin.param_state) diff --git a/src/lightning_app/frontend/panel/__init__.py b/src/lightning_app/frontend/panel/__init__.py new file mode 100644 index 0000000000000..cc2e6a884bb99 --- /dev/null +++ b/src/lightning_app/frontend/panel/__init__.py @@ -0,0 +1,5 @@ +from .panel_frontend import PanelFrontend + +__all__=[ + "PanelFrontend" +] \ No newline at end of file diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py new file mode 100644 index 0000000000000..649a38a00fc2f --- /dev/null +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -0,0 +1,83 @@ +"""The PanelFrontend wraps your Panel code in your LightningFlow""" +from __future__ import annotations + +import inspect +import logging +import pathlib +import subprocess +import sys +from typing import Callable + +from lightning_app.frontend.frontend import Frontend +from lightning_app.frontend.utilities.other import get_frontend_environment +from lightning_app.utilities.imports import requires +from lightning_app.utilities.log import get_frontend_logfile + +_logger = logging.getLogger("PanelFrontend") + + +class PanelFrontend(Frontend): + """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow + + To use this frontend, you must first install the `panel` package: + + .. code-block:: bash + + pip install panel + + Please note the Panel server will be logging output to error.log and output.log files + respectively. + + # Todo: Add Example + + Args: + render_fn: A pure function that contains your Panel code. This function must accept + exactly one argument, the `AppStateWatcher` object which you can use to get and + set variables in your flow (see example below). This function must return a + Panel Viewable. + + Raises: + TypeError: Raised if the render_fn is a class method + """ + + @requires("panel") + def __init__(self, render_fn: Callable): + # Todo: enable the render_fn to be a .py or .ipynb file + # Todo: enable the render_fn to not accept an AppStateWatcher as argument + super().__init__() + + if inspect.ismethod(render_fn): + raise TypeError( + "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a " + "pure function." + ) + + self.render_fn = render_fn + self._process: None | subprocess.Popen = None + _logger.debug("initialized") + + def start_server(self, host: str, port: int) -> None: + _logger.debug("starting server %s %s", host, port) + env = get_frontend_environment( + self.flow.name, + self.render_fn, + port, + host, + ) + std_err_out = get_frontend_logfile("error.log") + std_out_out = get_frontend_logfile("output.log") + with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: + self._process = subprocess.Popen( # pylint: disable=consider-using-with + [ + sys.executable, + pathlib.Path(__file__).parent / "panel_serve_render_fn.py", + ], + env=env, + stdout=stdout, + stderr=stderr, + ) + + def stop_server(self) -> None: + if self._process is None: + raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") + self._process.kill() diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py new file mode 100644 index 0000000000000..ecdb05cbd4277 --- /dev/null +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -0,0 +1,58 @@ +"""This file gets run by Python to lunch a Panel Server with Lightning. + +From here, we will call the render_fn that the user provided to the PanelFrontend. + +It requires the below environment variables to be set + +- LIGHTNING_FLOW_NAME +- LIGHTNING_RENDER_ADDRESS +- LIGHTNING_RENDER_FUNCTION +- LIGHTNING_RENDER_MODULE_FILE +- LIGHTNING_RENDER_PORT + +Example: + +.. code-block:: bash + + python panel_serve_render_fn + +""" +from __future__ import annotations + +import logging +import os + +import panel as pn + +from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher +from lightning_app.frontend.utilities.other import get_render_fn_from_environment + +_logger = logging.getLogger(__name__) + + +def _view(): + render_fn = get_render_fn_from_environment() + app = AppStateWatcher() + return render_fn(app) + + +def _get_websocket_origin() -> str: + # Todo: Improve this. I don't know how to find the specific host(s). + # I tried but it did not work in cloud + return "*" + + +def _serve(): + port = int(os.environ["LIGHTNING_RENDER_PORT"]) + address = os.environ["LIGHTNING_RENDER_ADDRESS"] + url = os.environ["LIGHTNING_FLOW_NAME"] + websocket_origin = _get_websocket_origin() + + pn.serve( + {url: _view}, address=address, port=port, websocket_origin=websocket_origin, show=False + ) + _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) + + +if __name__ == "__main__": + _serve() diff --git a/src/lightning_app/frontend/utilities/__init__.py b/src/lightning_app/frontend/utilities/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py new file mode 100644 index 0000000000000..5f7b71fd15b77 --- /dev/null +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -0,0 +1,83 @@ +"""The watch_app_state function enables us to trigger a callback function when ever the app +state changes""" +# Todo: Refactor with Streamlit +# Note: It would be nice one day to just watch changes within the Flow scope instead of whole app +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from typing import Callable + +import websockets + +from lightning_app.core.constants import APP_SERVER_PORT + +_logger = logging.getLogger(__name__) + +_CALLBACKS = [] +_THREAD: None | threading.Thread = None + + +def _get_ws_port(): + if "LIGHTNING_APP_STATE_URL" in os.environ: + return 8080 + return APP_SERVER_PORT + + +def _get_ws_url(): + port = _get_ws_port() + return f"ws://localhost:{port}/api/v1/ws" + + +def _run_callbacks(): + for callback in _CALLBACKS: + callback() + + +def _target_fn(): + async def update_fn(): + ws_url = _get_ws_url() + _logger.debug("connecting to web socket %s", ws_url) + async with websockets.connect(ws_url) as websocket: # pylint: disable=no-member + while True: + await websocket.recv() + # Note: I have not seen use cases where the two lines below are needed + # Note: Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # while (time.time() - last_updated) < 0.2: + # time.sleep(0.05) + _logger.debug("App State Changed. Running callbacks") + _run_callbacks() + + asyncio.run(update_fn()) + + +def _start_websocket(): + global _THREAD # pylint: disable=global-statement + if not _THREAD: + _logger.debug("starting thread") + _THREAD = threading.Thread(target=_target_fn) + _THREAD.setDaemon(True) + _THREAD.start() + _logger.debug("thread started") + + +def watch_app_state(callback: Callable): + """Start the process that serves the UI at the given hostname and port number. + + Arguments: + callback: A function to run when the app state changes. Must be thread safe. + + Example: + + .. code-block:: python + + def handle_state_change(): + print("The App State Changed") + watch_app_state(handle_state_change) + """ + + _CALLBACKS.append(callback) + + _start_websocket() diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py new file mode 100644 index 0000000000000..c2c3eef1e1feb --- /dev/null +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -0,0 +1,83 @@ +"""The AppStateWatcher enables a Frontend to + +- subscribe to app state changes +- to access and change the app state. + +This is particularly useful for the PanelFrontend but can be used by other Frontends too. +""" +from __future__ import annotations + +import logging + +import param + +from lightning_app.frontend.utilities.app_state_comm import watch_app_state +from lightning_app.frontend.utilities.other import get_flow_state +from lightning_app.utilities.imports import requires +from lightning_app.utilities.state import AppState + +_logger = logging.getLogger(__name__) + + +class AppStateWatcher(param.Parameterized): + """The AppStateWatcher enables a Frontend to + + - subscribe to app state changes + - to access and change the app state. + + This is particularly useful for the PanelFrontend, but can be used by + other Frontends too. + + Example: + + .. code-block:: python + + import param + app = AppStateWatcher() + + app.state.counter = 1 + + @param.depends(app.param.state, watch=True) + def update(state): + print(f"The counter was updated to {state.counter}") + + app.state.counter += 1 + + This would print 'The counter was updated to 2'. + + The AppStateWatcher is build on top of Param which is a framework like dataclass, attrs and + Pydantic which additionally provides powerful and unique features for building reactive apps. + """ + + state: AppState = param.ClassSelector( + class_=AppState, + doc=""" + The AppState holds the state of the app reduced to the scope of the Flow""", + ) + + def __new__(cls): + # This makes the AppStateWatcher a *singleton*. + # The AppStateWatcher is a singleton to minimize the number of requests etc.. + if not hasattr(cls, "instance"): + cls.instance = super(AppStateWatcher, cls).__new__(cls) + return cls.instance + + @requires("param") + def __init__(self): + super().__init__() + self._start_watching() + + def _start_watching(self): + watch_app_state(self._handle_state_changed) + self._request_state() + + def _get_flow_state(self): + return get_flow_state() + + def _request_state(self): + self.state = self._get_flow_state() + _logger.debug("Request app state") + + def _handle_state_changed(self): + _logger.debug("Handle app state changed") + self._request_state() diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py new file mode 100644 index 0000000000000..6001a195c5084 --- /dev/null +++ b/src/lightning_app/frontend/utilities/other.py @@ -0,0 +1,64 @@ +"""Utility functions for lightning Frontends""" +# Todo: Refactor stream_lit and streamlit_base to use this functionality + +from __future__ import annotations + +import inspect +import os +import pydoc +from typing import Callable, Union + +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState + + +def get_render_fn_from_environment() -> Callable: + """Returns the render_fn function to serve in the Frontend""" + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + module = pydoc.importfile(render_fn_module_file) + return getattr(module, render_fn_name) + + +def _reduce_to_flow_scope(state: AppState, flow: Union[str, LightningFlow]) -> AppState: + """Returns a new AppState with the scope reduced to the given flow""" + flow_name = flow.name if isinstance(flow, LightningFlow) else flow + flow_name_parts = flow_name.split(".")[1:] # exclude root + flow_state = state + for part in flow_name_parts: + flow_state = getattr(flow_state, part) + return flow_state + + +def get_flow_state() -> AppState: + """Returns an AppState scoped to the current Flow + + Returns: + AppState: An AppState scoped to the current Flow. + """ + app_state = AppState() + app_state._request_state() # pylint: disable=protected-access + flow = os.environ["LIGHTNING_FLOW_NAME"] + flow_state = _reduce_to_flow_scope(app_state, flow) + return flow_state + + +def get_frontend_environment(flow: str, render_fn: Callable, port: int, host: str) -> os._Environ: + """Returns an _Environ with the environment variables for serving a Frontend app set. + + Args: + flow (str): The name of the flow, for example root.lit_frontend + render_fn (Callable): A function to render + port (int): The port number, for example 54321 + host (str): The host, for example 'localhost' + + Returns: + os._Environ: An environement + """ + env = os.environ.copy() + env["LIGHTNING_FLOW_NAME"] = flow + env["LIGHTNING_RENDER_FUNCTION"] = render_fn.__name__ + env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn).__file__ + env["LIGHTNING_RENDER_PORT"] = str(port) + env["LIGHTNING_RENDER_ADDRESS"] = str(host) + return env diff --git a/src/lightning_app/utilities/state.py b/src/lightning_app/utilities/state.py index 0802a426e7349..a25a7fcf01f98 100644 --- a/src/lightning_app/utilities/state.py +++ b/src/lightning_app/utilities/state.py @@ -1,231 +1,231 @@ -import enum -import json -import logging -import os -from copy import deepcopy -from typing import Any, Dict, Optional, Tuple, Union - -from deepdiff import DeepDiff -from requests import Session -from requests.exceptions import ConnectionError - -from lightning_app.core.constants import APP_SERVER_HOST, APP_SERVER_PORT -from lightning_app.storage.drive import _maybe_create_drive -from lightning_app.utilities.app_helpers import AppStatePlugin, BaseStatePlugin -from lightning_app.utilities.network import _configure_session - -logger = logging.getLogger(__name__) - -# GLOBAL APP STATE -_LAST_STATE = None -_STATE = None - - -class AppStateType(enum.Enum): - STREAMLIT = enum.auto() - DEFAULT = enum.auto() - - -def headers_for(context: Dict[str, str]) -> Dict[str, str]: - return { - "X-Lightning-Session-UUID": context.get("token", ""), - "X-Lightning-Session-ID": context.get("session_id", ""), - "X-Lightning-Type": context.get("type", ""), - } - - -class AppState: - - _APP_PRIVATE_KEYS: Tuple[str, ...] = ( - "_host", - "_session_id", - "_state", - "_last_state", - "_url", - "_port", - "_request_state", - "_store_state", - "_send_state", - "_my_affiliation", - "_find_state_under_affiliation", - "_plugin", - "_attach_plugin", - "_authorized", - "is_authorized", - "_debug", - "_session", - ) - _MY_AFFILIATION: Tuple[str, ...] = () - - def __init__( - self, - host: Optional[str] = None, - port: Optional[int] = None, - last_state: Optional[Dict] = None, - state: Optional[Dict] = None, - my_affiliation: Tuple[str, ...] = None, - plugin: Optional[BaseStatePlugin] = None, - ) -> None: - """The AppState class enable streamlit user to interact their application state. - - When the state isn't defined, it would be pulled from the app REST API Server. - If the state gets modified by the user, the new state would be sent to the API Server. - - Arguments: - host: Rest API Server current host - port: Rest API Server current port - last_state: The state pulled on first access. - state: The state modified by the user. - my_affiliation: A tuple describing the affiliation this app state represents. When storing a state dict - on this AppState, this affiliation will be used to reduce the scope of the given state. - plugin: A plugin to handle authorization. - """ - use_localhost = "LIGHTNING_APP_STATE_URL" not in os.environ - self._host = host or APP_SERVER_HOST - self._port = port or (APP_SERVER_PORT if use_localhost else None) - self._url = f"{self._host}:{self._port}" if use_localhost else self._host - self._last_state = last_state - self._state = state - self._session_id = "1234" - self._my_affiliation = my_affiliation if my_affiliation is not None else AppState._MY_AFFILIATION - self._authorized = None - self._attach_plugin(plugin) - self._session = self._configure_session() - - def _attach_plugin(self, plugin: Optional[BaseStatePlugin]) -> None: - if plugin is not None: - plugin = plugin - else: - plugin = AppStatePlugin() - self._plugin = plugin - - @staticmethod - def _find_state_under_affiliation(state, my_affiliation: Tuple[str, ...]) -> Dict[str, Any]: - """This method is used to extract the subset of the app state associated with the given affiliation. - - For example, if the affiliation is ``("root", "subflow")``, then the returned state will be - ``state["flows"]["subflow"]``. - """ - children_state = state - for name in my_affiliation: - if name in children_state["flows"]: - children_state = children_state["flows"][name] - elif name in children_state["works"]: - children_state = children_state["works"][name] - else: - raise ValueError(f"Failed to extract the state under the affiliation '{my_affiliation}'.") - return children_state - - def _store_state(self, state: Dict[str, Any]) -> None: - # Relying on the global variable to ensure the - # deep_diff is done on the entire state. - global _LAST_STATE - global _STATE - _LAST_STATE = deepcopy(state) - _STATE = state - # If the affiliation is passed, the AppState was created in a LightningFlow context. - # The state should be only the one of this LightningFlow and its children. - self._last_state = self._find_state_under_affiliation(_LAST_STATE, self._my_affiliation) - self._state = self._find_state_under_affiliation(_STATE, self._my_affiliation) - - def send_delta(self) -> None: - app_url = f"{self._url}/api/v1/delta" - deep_diff = DeepDiff(_LAST_STATE, _STATE) - assert self._plugin is not None - # TODO: Find how to prevent the infinite loop on refresh without storing the DeepDiff - if self._plugin.should_update_app(deep_diff): - data = {"delta": json.loads(deep_diff.to_json())} - headers = headers_for(self._plugin.get_context()) - try: - # TODO: Send the delta directly to the REST API. - response = self._session.post(app_url, json=data, headers=headers) - except ConnectionError as e: - raise AttributeError("Failed to connect and send the app state. Is the app running?") from e - - if response and response.status_code != 200: - raise Exception(f"The response from the server was {response.status_code}. Your inputs were rejected.") - - def _request_state(self) -> None: - if self._state is not None: - return - app_url = f"{self._url}/api/v1/state" - headers = headers_for(self._plugin.get_context()) if self._plugin else {} - try: - response = self._session.get(app_url, headers=headers, timeout=1) - except ConnectionError as e: - raise AttributeError("Failed to connect and fetch the app state. Is the app running?") from e - - self._authorized = response.status_code - if self._authorized != 200: - return - logger.debug(f"GET STATE {response} {response.json()}") - self._store_state(response.json()) - - def __getattr__(self, name: str) -> Union[Any, "AppState"]: - if name in self._APP_PRIVATE_KEYS: - return object.__getattr__(self, name) - - # The state needs to be fetched on access if it doesn't exist. - self._request_state() - - if name in self._state["vars"]: - value = self._state["vars"][name] - if isinstance(value, dict): - return _maybe_create_drive("root." + ".".join(self._my_affiliation), value) - return value - - elif name in self._state.get("works", {}): - return AppState( - self._host, self._port, last_state=self._last_state["works"][name], state=self._state["works"][name] - ) - - elif name in self._state.get("flows", {}): - return AppState( - self._host, - self._port, - last_state=self._last_state["flows"][name], - state=self._state["flows"][name], - ) - - raise AttributeError( - f"Failed to access '{name}' through `AppState`. The state provides:" - f" Variables: {list(self._state['vars'].keys())}," - f" Components: {list(self._state.get('flows', {}).keys()) + list(self._state.get('works', {}).keys())}", - ) - - def __setattr__(self, name: str, value: Any) -> None: - if name in self._APP_PRIVATE_KEYS: - object.__setattr__(self, name, value) - return - - # The state needs to be fetched on access if it doesn't exist. - self._request_state() - - # TODO: Find a way to aggregate deltas to avoid making - # request for each attribute change. - if name in self._state["vars"]: - self._state["vars"][name] = value - self.send_delta() - - elif name in self._state["flows"]: - raise AttributeError("You shouldn't set the flows directly onto the state. Use its attributes instead.") - - elif name in self._state["works"]: - raise AttributeError("You shouldn't set the works directly onto the state. Use its attributes instead.") - - else: - raise AttributeError( - f"Failed to access '{name}' through `AppState`. The state provides:" - f" Variables: {list(self._state['vars'].keys())}," - f" Components: {list(self._state['flows'].keys()) + list(self._state['works'].keys())}", - ) - - def __repr__(self) -> str: - return str(self._state) - - def __bool__(self) -> bool: - return bool(self._state) - - @staticmethod - def _configure_session() -> Session: - return _configure_session() +import enum +import json +import logging +import os +from copy import deepcopy +from typing import Any, Dict, Optional, Tuple, Union + +from deepdiff import DeepDiff +from requests import Session +from requests.exceptions import ConnectionError + +from lightning_app.core.constants import APP_SERVER_HOST, APP_SERVER_PORT +from lightning_app.storage.drive import _maybe_create_drive +from lightning_app.utilities.app_helpers import AppStatePlugin, BaseStatePlugin +from lightning_app.utilities.network import _configure_session + +logger = logging.getLogger(__name__) + +# GLOBAL APP STATE +_LAST_STATE = None +_STATE = None + + +class AppStateType(enum.Enum): + STREAMLIT = enum.auto() + DEFAULT = enum.auto() + + +def headers_for(context: Dict[str, str]) -> Dict[str, str]: + return { + "X-Lightning-Session-UUID": context.get("token", ""), + "X-Lightning-Session-ID": context.get("session_id", ""), + "X-Lightning-Type": context.get("type", ""), + } + + +class AppState: + + _APP_PRIVATE_KEYS: Tuple[str, ...] = ( + "_host", + "_session_id", + "_state", + "_last_state", + "_url", + "_port", + "_request_state", + "_store_state", + "_send_state", + "_my_affiliation", + "_find_state_under_affiliation", + "_plugin", + "_attach_plugin", + "_authorized", + "is_authorized", + "_debug", + "_session", + ) + _MY_AFFILIATION: Tuple[str, ...] = () + + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + last_state: Optional[Dict] = None, + state: Optional[Dict] = None, + my_affiliation: Tuple[str, ...] = None, + plugin: Optional[BaseStatePlugin] = None, + ) -> None: + """The AppState class enable Frontend users to interact with their application state. + + When the state isn't defined, it would be pulled from the app REST API Server. + If the state gets modified by the user, the new state would be sent to the API Server. + + Arguments: + host: Rest API Server current host + port: Rest API Server current port + last_state: The state pulled on first access. + state: The state modified by the user. + my_affiliation: A tuple describing the affiliation this app state represents. When storing a state dict + on this AppState, this affiliation will be used to reduce the scope of the given state. + plugin: A plugin to handle authorization. + """ + use_localhost = "LIGHTNING_APP_STATE_URL" not in os.environ + self._host = host or APP_SERVER_HOST + self._port = port or (APP_SERVER_PORT if use_localhost else None) + self._url = f"{self._host}:{self._port}" if use_localhost else self._host + self._last_state = last_state + self._state = state + self._session_id = "1234" + self._my_affiliation = my_affiliation if my_affiliation is not None else AppState._MY_AFFILIATION + self._authorized = None + self._attach_plugin(plugin) + self._session = self._configure_session() + + def _attach_plugin(self, plugin: Optional[BaseStatePlugin]) -> None: + if plugin is not None: + plugin = plugin + else: + plugin = AppStatePlugin() + self._plugin = plugin + + @staticmethod + def _find_state_under_affiliation(state, my_affiliation: Tuple[str, ...]) -> Dict[str, Any]: + """This method is used to extract the subset of the app state associated with the given affiliation. + + For example, if the affiliation is ``("root", "subflow")``, then the returned state will be + ``state["flows"]["subflow"]``. + """ + children_state = state + for name in my_affiliation: + if name in children_state["flows"]: + children_state = children_state["flows"][name] + elif name in children_state["works"]: + children_state = children_state["works"][name] + else: + raise ValueError(f"Failed to extract the state under the affiliation '{my_affiliation}'.") + return children_state + + def _store_state(self, state: Dict[str, Any]) -> None: + # Relying on the global variable to ensure the + # deep_diff is done on the entire state. + global _LAST_STATE + global _STATE + _LAST_STATE = deepcopy(state) + _STATE = state + # If the affiliation is passed, the AppState was created in a LightningFlow context. + # The state should be only the one of this LightningFlow and its children. + self._last_state = self._find_state_under_affiliation(_LAST_STATE, self._my_affiliation) + self._state = self._find_state_under_affiliation(_STATE, self._my_affiliation) + + def send_delta(self) -> None: + app_url = f"{self._url}/api/v1/delta" + deep_diff = DeepDiff(_LAST_STATE, _STATE) + assert self._plugin is not None + # TODO: Find how to prevent the infinite loop on refresh without storing the DeepDiff + if self._plugin.should_update_app(deep_diff): + data = {"delta": json.loads(deep_diff.to_json())} + headers = headers_for(self._plugin.get_context()) + try: + # TODO: Send the delta directly to the REST API. + response = self._session.post(app_url, json=data, headers=headers) + except ConnectionError as e: + raise AttributeError("Failed to connect and send the app state. Is the app running?") from e + + if response and response.status_code != 200: + raise Exception(f"The response from the server was {response.status_code}. Your inputs were rejected.") + + def _request_state(self) -> None: + if self._state is not None: + return + app_url = f"{self._url}/api/v1/state" + headers = headers_for(self._plugin.get_context()) if self._plugin else {} + try: + response = self._session.get(app_url, headers=headers, timeout=1) + except ConnectionError as e: + raise AttributeError("Failed to connect and fetch the app state. Is the app running?") from e + + self._authorized = response.status_code + if self._authorized != 200: + return + logger.debug(f"GET STATE {response} {response.json()}") + self._store_state(response.json()) + + def __getattr__(self, name: str) -> Union[Any, "AppState"]: + if name in self._APP_PRIVATE_KEYS: + return object.__getattr__(self, name) + + # The state needs to be fetched on access if it doesn't exist. + self._request_state() + + if name in self._state["vars"]: + value = self._state["vars"][name] + if isinstance(value, dict): + return _maybe_create_drive("root." + ".".join(self._my_affiliation), value) + return value + + elif name in self._state.get("works", {}): + return AppState( + self._host, self._port, last_state=self._last_state["works"][name], state=self._state["works"][name] + ) + + elif name in self._state.get("flows", {}): + return AppState( + self._host, + self._port, + last_state=self._last_state["flows"][name], + state=self._state["flows"][name], + ) + + raise AttributeError( + f"Failed to access '{name}' through `AppState`. The state provides:" + f" Variables: {list(self._state['vars'].keys())}," + f" Components: {list(self._state.get('flows', {}).keys()) + list(self._state.get('works', {}).keys())}", + ) + + def __setattr__(self, name: str, value: Any) -> None: + if name in self._APP_PRIVATE_KEYS: + object.__setattr__(self, name, value) + return + + # The state needs to be fetched on access if it doesn't exist. + self._request_state() + + # TODO: Find a way to aggregate deltas to avoid making + # request for each attribute change. + if name in self._state["vars"]: + self._state["vars"][name] = value + self.send_delta() + + elif name in self._state["flows"]: + raise AttributeError("You shouldn't set the flows directly onto the state. Use its attributes instead.") + + elif name in self._state["works"]: + raise AttributeError("You shouldn't set the works directly onto the state. Use its attributes instead.") + + else: + raise AttributeError( + f"Failed to access '{name}' through `AppState`. The state provides:" + f" Variables: {list(self._state['vars'].keys())}," + f" Components: {list(self._state['flows'].keys()) + list(self._state['works'].keys())}", + ) + + def __repr__(self) -> str: + return str(self._state) + + def __bool__(self) -> bool: + return bool(self._state) + + @staticmethod + def _configure_session() -> Session: + return _configure_session() diff --git a/tests/tests_app/frontend/panel/__init__.py b/tests/tests_app/frontend/panel/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py new file mode 100644 index 0000000000000..be3c625ee251e --- /dev/null +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -0,0 +1,94 @@ +import os +import runpy +import sys +from unittest import mock +from unittest.mock import Mock + +import pytest + +from lightning_app import LightningFlow +from lightning_app.frontend.panel import PanelFrontend +from lightning_app.utilities.state import AppState + + +def test_stop_server_not_running(): + """If the server is not running but stopped an Exception should be + raised""" + frontend = PanelFrontend(render_fn=Mock()) + with pytest.raises(RuntimeError, match="Server is not running."): + frontend.stop_server() + + +def _noop_render_fn(_): + pass + + +class MockFlow(LightningFlow): + @property + def name(self): + return "root.my.flow" + + def run(self): + pass + + +@mock.patch("lightning_app.frontend.panel.panel_frontend.subprocess") +def test_streamlit_frontend_start_stop_server(subprocess_mock): + """Test that `PanelFrontend.start_server()` invokes subprocess.Popen with the right parameters.""" + # Given + frontend = PanelFrontend(render_fn=_noop_render_fn) + frontend.flow = MockFlow() + # When + frontend.start_server(host="hostname", port=1111) + # Then + subprocess_mock.Popen.assert_called_once() + + env_variables = subprocess_mock.method_calls[0].kwargs["env"] + call_args = subprocess_mock.method_calls[0].args[0] + assert call_args[0] == sys.executable + assert call_args[1].exists() + assert str(call_args[1]).endswith("panel_serve_render_fn.py") + assert len(call_args)==2 + + assert env_variables["LIGHTNING_FLOW_NAME"] == "root.my.flow" + assert env_variables["LIGHTNING_RENDER_ADDRESS"] == "hostname" + assert env_variables["LIGHTNING_RENDER_FUNCTION"] == "_noop_render_fn" + assert env_variables["LIGHTNING_RENDER_MODULE_FILE"] == __file__ + assert env_variables["LIGHTNING_RENDER_PORT"] == "1111" + + assert "LIGHTNING_FLOW_NAME" not in os.environ + assert "LIGHTNING_RENDER_FUNCTION" not in os.environ + assert "LIGHTNING_RENDER_MODULE_FILE" not in os.environ + assert "LIGHTNING_RENDER_MODULE_PORT" not in os.environ + assert "LIGHTNING_RENDER_MODULE_ADDRESS" not in os.environ + # When + frontend.stop_server() + # Then + subprocess_mock.Popen().kill.assert_called_once() + +def _call_me(state): + assert isinstance(state, AppState) + + +@mock.patch.dict( + os.environ, + { + "LIGHTNING_FLOW_NAME": "root", + "LIGHTNING_RENDER_FUNCTION": "_call_me", + "LIGHTNING_RENDER_MODULE_FILE": __file__, + "LIGHTNING_RENDER_ADDRESS": "127.0.0.1", + "LIGHTNING_RENDER_PORT": "61896", + }, +) +def test_panel_wrapper_calls_render_fn(*_): + runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn") + # TODO: find a way to assert that _call_me got called + + +def test_method_exception(): + class A: + def render_fn(self): + pass + + with pytest.raises(TypeError, match="being a method"): + PanelFrontend(render_fn=A().render_fn) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py new file mode 100644 index 0000000000000..9135995814a78 --- /dev/null +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -0,0 +1,59 @@ +"""This panel_serve_render_fn file gets run by Python to lunch a Panel Server with Lightning.""" +import os +from unittest import mock + +import pytest + +from lightning_app.frontend.panel.panel_serve_render_fn import _serve, _view +from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher + + +@pytest.fixture(autouse=True, scope="module") +def mock_settings_env_vars(): + """Set the LIGHTNING environment variables""" + with mock.patch.dict( + os.environ, + { + "LIGHTNING_FLOW_NAME": "root.lit_panel", + "LIGHTNING_RENDER_ADDRESS": "localhost", + "LIGHTNING_RENDER_FUNCTION": "render_fn", + "LIGHTNING_RENDER_MODULE_FILE": __file__, + "LIGHTNING_RENDER_PORT": "61896", + }, + ): + yield + + +def do_nothing(_): + """Be lazy!""" + + +@pytest.fixture(autouse=True, scope="module") +def mock_request_state(): + """Avoid requests to the api""" + with mock.patch( + "lightning_app.frontend.utilities.app_state_watcher.AppStateWatcher._start_watching", + do_nothing, + ): + yield + + +def render_fn(app): + """Test function that just passes through the app""" + return app + + +def test_view(): + """We have a helper _view function that provides the AppStateWatcher as argument to render_fn + and returns the result""" + result = _view() + assert isinstance(result, AppStateWatcher) + + +@mock.patch("panel.serve") +def test_serve(pn_serve: mock.MagicMock): + """We can run python panel_serve_render_fn to serve the render_fn""" + _serve() + pn_serve.assert_called_once_with( + {"root.lit_panel": _view}, address="localhost", port=61896, websocket_origin="*", show=False + ) diff --git a/tests/tests_app/frontend/utilities/__init__.py b/tests/tests_app/frontend/utilities/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/tests_app/frontend/utilities/conftest.py b/tests/tests_app/frontend/utilities/conftest.py new file mode 100644 index 0000000000000..48d903699ed07 --- /dev/null +++ b/tests/tests_app/frontend/utilities/conftest.py @@ -0,0 +1,81 @@ +from re import L +import pytest +from unittest import mock +import os + +FLOW_SUB= "lit_panel" +FLOW = f"root.{FLOW_SUB}" +PORT = 61896 + +FLOW_STATE = { + "vars": { + "_paths": {}, + "_layout": {"target": f"http://localhost:{PORT}/{FLOW}"}, + }, + "calls": {}, + "flows": {}, + "works": {}, + "structures": {}, + "changes": {}, +} + +APP_STATE = { + "vars": {"_paths": {}, "_layout": [{"name": "home", "content": FLOW}]}, + "calls": {}, + "flows": { + FLOW_SUB: FLOW_STATE, + }, + "works": {}, + "structures": {}, + "changes": {}, + "app_state": {"stage": "running"}, +} + +def _request_state(self): + _state = APP_STATE + self._store_state(_state) + +def render_fn(app): + """Test function that just passes through the app""" + return app + +@pytest.fixture(autouse=True, scope="module") +def mock_request_state(): + """Avoid requests to the api""" + with mock.patch("lightning_app.utilities.state.AppState._request_state", _request_state): + yield + +@pytest.fixture(autouse=True, scope="module") +def mock_settings_env_vars(): + """Set the LIGHTNING environment variables""" + with mock.patch.dict( + os.environ, + { + "LIGHTNING_FLOW_NAME": FLOW, + "LIGHTNING_RENDER_ADDRESS": "localhost", + "LIGHTNING_RENDER_FUNCTION": "render_fn", + "LIGHTNING_RENDER_MODULE_FILE": __file__, + "LIGHTNING_RENDER_PORT": f"{PORT}", + }, + ): + yield + +def do_nothing(): + """Be lazy!""" + + +@pytest.fixture(autouse=True, scope="module") +def mock_start_websocket(): + """Avoid starting the websocket""" + with mock.patch( + "lightning_app.frontend.utilities.app_state_comm._start_websocket", do_nothing + ): + yield + +@pytest.fixture +def app_state_state(): + return APP_STATE.copy() + +@pytest.fixture +def flow_state_state(): + return FLOW_STATE.copy() \ No newline at end of file diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py new file mode 100644 index 0000000000000..cccc453d13310 --- /dev/null +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -0,0 +1,40 @@ +"""The watch_app_state function enables us to trigger a callback function when ever the app +state changes""" +import os +from unittest import mock + +from lightning_app.core.constants import APP_SERVER_PORT +from lightning_app.frontend.utilities.app_state_comm import ( + _get_ws_url, + _run_callbacks, + watch_app_state, +) + + +def do_nothing(): + """Be lazy!""" + + +def test_get_ws_url_when_local(): + """The websocket uses port APP_SERVER_PORT when local""" + assert _get_ws_url() == f"ws://localhost:{APP_SERVER_PORT}/api/v1/ws" + + +@mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "some_url"}) +def test_get_ws_url_when_cloud(): + """The websocket uses port 8080 when LIGHTNING_APP_STATE_URL is set""" + assert _get_ws_url() == "ws://localhost:8080/api/v1/ws" + + +def test_watch_app_state(): + """We can watch the app state and run a callback function when it changes""" + callback = mock.MagicMock() + # When + watch_app_state(callback) + + # Here we would like to send messages via the web socket + # For testing the web socket is not started. See conftest.py + # So we need to manually trigger _run_callbacks here + _run_callbacks() + # Then + callback.assert_called_once() \ No newline at end of file diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py new file mode 100644 index 0000000000000..f4f5b173b7397 --- /dev/null +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -0,0 +1,36 @@ +"""The AppStateWatcher enables a Frontend to + +- subscribe to app state changes +- to access and change the app state. + +This is particularly useful for the PanelFrontend but can be used by other Frontends too. +""" +# pylint: disable=protected-access +from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher +from lightning_app.utilities.state import AppState + +def test_init(flow_state_state: dict): + """We can instantiate the AppStateWatcher + + - the .state is set + - the .state is scoped to the flow state + """ + app = AppStateWatcher() + assert isinstance(app.state, AppState) + assert app.state._state == flow_state_state + + +def test_handle_state_changed(flow_state_state: dict): + """We can handle state changes by updating the state""" + app = AppStateWatcher() + org_state = app.state + app._handle_state_changed() + assert app.state is not org_state + assert app.state._state == flow_state_state + + +def test_is_singleton(): + """The AppStateWatcher is a singleton for efficiency reasons""" + watcher1 = AppStateWatcher() + watcher2 = AppStateWatcher() + assert watcher1 is watcher2 diff --git a/tests/tests_app/frontend/utilities/test_other.py b/tests/tests_app/frontend/utilities/test_other.py new file mode 100644 index 0000000000000..79f2c947803ba --- /dev/null +++ b/tests/tests_app/frontend/utilities/test_other.py @@ -0,0 +1,46 @@ +"""We have some utility functions that can be used across frontends""" +import inspect +import os + +from lightning_app.frontend.utilities.other import ( + get_flow_state, + get_frontend_environment, + get_render_fn_from_environment, +) +from lightning_app.utilities.state import AppState + + +def test_get_flow_state(flow_state_state: dict): + """We have a method to get an AppState scoped to the Flow state""" + # When + flow_state = get_flow_state() + # Then + assert isinstance(flow_state, AppState) + assert flow_state._state == flow_state_state # pylint: disable=protected-access + + +def test_get_render_fn_from_environment(): + """We have a method to get the render_fn from the environment""" + # When + render_fn = get_render_fn_from_environment() + # Then + assert inspect.getfile(render_fn) == os.environ["LIGHTNING_RENDER_MODULE_FILE"] + assert render_fn.__name__ == os.environ["LIGHTNING_RENDER_FUNCTION"] + + +def some_fn(_): + """Be lazy!""" + + +def test__get_frontend_environment(): + """We have a utility function to get the frontend render_fn environment""" + # When + env = get_frontend_environment( + flow="root.lit_frontend", render_fn=some_fn, host="myhost", port=1234 + ) + # Then + assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" + assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" + assert env["LIGHTNING_RENDER_FUNCTION"] == "some_fn" + assert env["LIGHTNING_RENDER_MODULE_FILE"] == __file__ + assert env["LIGHTNING_RENDER_PORT"] == "1234" From cbff6ec07136101de2d2f3d5eacf53c2f6661e02 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 9 Jul 2022 11:13:08 +0000 Subject: [PATCH 007/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 2 +- .../examples/app_interact_from_frontend.py | 16 +++++++---- .../panel/examples/app_streamlit.py | 5 +++- src/lightning_app/frontend/panel/__init__.py | 4 +-- .../frontend/panel/panel_frontend.py | 9 +++--- .../frontend/panel/panel_serve_render_fn.py | 5 +--- .../frontend/utilities/app_state_comm.py | 3 +- .../frontend/utilities/app_state_watcher.py | 11 +++++--- src/lightning_app/frontend/utilities/other.py | 12 ++++---- .../frontend/panel/test_panel_frontend.py | 8 +++--- .../panel/test_panel_serve_render_fn.py | 12 ++++---- .../tests_app/frontend/utilities/conftest.py | 28 +++++++++++-------- .../frontend/utilities/test_app_state_comm.py | 19 +++++-------- .../utilities/test_app_state_watcher.py | 9 +++--- .../frontend/utilities/test_other.py | 14 ++++------ 15 files changed, 80 insertions(+), 77 deletions(-) diff --git a/.gitignore b/.gitignore index 71ad77140b434..556905e208e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ tags .tags # Lightning StreamlitFrontend or PanelFrontend -.storage \ No newline at end of file +.storage diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py index e7d965ed3241e..4f37fc53dc2bd 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py @@ -6,32 +6,35 @@ pn.extension(sizing_mode="stretch_width") + def your_panel_app(app: AppStateWatcher): - + submit_button = pn.widgets.Button(name="submit") - + @pn.depends(submit_button, watch=True) def submit(_): - app.state.count += 1 + app.state.count += 1 @pn.depends(app.param.state) def current_count(_): - return f'current count: {app.state.count}' + return f"current count: {app.state.count}" return pn.Column( submit_button, current_count, ) + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() self._frontend = PanelFrontend(render_fn=your_panel_app) - self.count=0 - + self.count = 0 + def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -40,4 +43,5 @@ def __init__(self): def configure_layout(self): return {"name": "home", "content": self.lit_panel} + app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py index 120b207d66dfd..b06fb09857258 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py @@ -8,14 +8,16 @@ def your_streamlit_app(lightning_app_state): - st.write('hello world') + st.write("hello world") st.write(lightning_app_state) st.write(os.environ["LIGHTNING_FLOW_NAME"]) + class LitStreamlit(L.LightningFlow): def configure_layout(self): return StreamlitFrontend(render_fn=your_streamlit_app) + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -25,4 +27,5 @@ def configure_layout(self): tab1 = {"name": "home", "content": self.lit_streamlit} return tab1 + app = L.LightningApp(LitApp()) diff --git a/src/lightning_app/frontend/panel/__init__.py b/src/lightning_app/frontend/panel/__init__.py index 6381e7abac1c4..59b26d49ebacd 100644 --- a/src/lightning_app/frontend/panel/__init__.py +++ b/src/lightning_app/frontend/panel/__init__.py @@ -1,6 +1,4 @@ from lightning_app.frontend.panel.panel_frontend import PanelFrontend from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher -__all__=[ - "PanelFrontend", "AppStateWatcher" -] +__all__ = ["PanelFrontend", "AppStateWatcher"] diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 649a38a00fc2f..9b5874e42d5a1 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -1,4 +1,4 @@ -"""The PanelFrontend wraps your Panel code in your LightningFlow""" +"""The PanelFrontend wraps your Panel code in your LightningFlow.""" from __future__ import annotations import inspect @@ -17,7 +17,7 @@ class PanelFrontend(Frontend): - """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow + """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. To use this frontend, you must first install the `panel` package: @@ -48,8 +48,7 @@ def __init__(self, render_fn: Callable): if inspect.ismethod(render_fn): raise TypeError( - "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a " - "pure function." + "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a " "pure function." ) self.render_fn = render_fn @@ -67,7 +66,7 @@ def start_server(self, host: str, port: int) -> None: std_err_out = get_frontend_logfile("error.log") std_out_out = get_frontend_logfile("output.log") with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: - self._process = subprocess.Popen( # pylint: disable=consider-using-with + self._process = subprocess.Popen( # pylint: disable=consider-using-with [ sys.executable, pathlib.Path(__file__).parent / "panel_serve_render_fn.py", diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py index ecdb05cbd4277..64db99664247c 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -15,7 +15,6 @@ .. code-block:: bash python panel_serve_render_fn - """ from __future__ import annotations @@ -48,9 +47,7 @@ def _serve(): url = os.environ["LIGHTNING_FLOW_NAME"] websocket_origin = _get_websocket_origin() - pn.serve( - {url: _view}, address=address, port=port, websocket_origin=websocket_origin, show=False - ) + pn.serve({url: _view}, address=address, port=port, websocket_origin=websocket_origin, show=False) _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 5f7b71fd15b77..d407b2c1d58f2 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -1,5 +1,4 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app -state changes""" +"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" # Todo: Refactor with Streamlit # Note: It would be nice one day to just watch changes within the Flow scope instead of whole app from __future__ import annotations diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index c2c3eef1e1feb..35ad8926f83a0 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -1,4 +1,4 @@ -"""The AppStateWatcher enables a Frontend to +"""The AppStateWatcher enables a Frontend to. - subscribe to app state changes - to access and change the app state. @@ -20,7 +20,7 @@ class AppStateWatcher(param.Parameterized): - """The AppStateWatcher enables a Frontend to + """The AppStateWatcher enables a Frontend to. - subscribe to app state changes - to access and change the app state. @@ -33,14 +33,17 @@ class AppStateWatcher(param.Parameterized): .. code-block:: python import param + app = AppStateWatcher() app.state.counter = 1 + @param.depends(app.param.state, watch=True) def update(state): print(f"The counter was updated to {state.counter}") + app.state.counter += 1 This would print 'The counter was updated to 2'. @@ -59,9 +62,9 @@ def __new__(cls): # This makes the AppStateWatcher a *singleton*. # The AppStateWatcher is a singleton to minimize the number of requests etc.. if not hasattr(cls, "instance"): - cls.instance = super(AppStateWatcher, cls).__new__(cls) + cls.instance = super().__new__(cls) return cls.instance - + @requires("param") def __init__(self): super().__init__() diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py index 6001a195c5084..fdfb8ef6ec40f 100644 --- a/src/lightning_app/frontend/utilities/other.py +++ b/src/lightning_app/frontend/utilities/other.py @@ -1,4 +1,4 @@ -"""Utility functions for lightning Frontends""" +"""Utility functions for lightning Frontends.""" # Todo: Refactor stream_lit and streamlit_base to use this functionality from __future__ import annotations @@ -13,15 +13,15 @@ def get_render_fn_from_environment() -> Callable: - """Returns the render_fn function to serve in the Frontend""" + """Returns the render_fn function to serve in the Frontend.""" render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] module = pydoc.importfile(render_fn_module_file) return getattr(module, render_fn_name) -def _reduce_to_flow_scope(state: AppState, flow: Union[str, LightningFlow]) -> AppState: - """Returns a new AppState with the scope reduced to the given flow""" +def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppState: + """Returns a new AppState with the scope reduced to the given flow.""" flow_name = flow.name if isinstance(flow, LightningFlow) else flow flow_name_parts = flow_name.split(".")[1:] # exclude root flow_state = state @@ -31,13 +31,13 @@ def _reduce_to_flow_scope(state: AppState, flow: Union[str, LightningFlow]) -> A def get_flow_state() -> AppState: - """Returns an AppState scoped to the current Flow + """Returns an AppState scoped to the current Flow. Returns: AppState: An AppState scoped to the current Flow. """ app_state = AppState() - app_state._request_state() # pylint: disable=protected-access + app_state._request_state() # pylint: disable=protected-access flow = os.environ["LIGHTNING_FLOW_NAME"] flow_state = _reduce_to_flow_scope(app_state, flow) return flow_state diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index be3c625ee251e..c22f559875dbf 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -12,8 +12,7 @@ def test_stop_server_not_running(): - """If the server is not running but stopped an Exception should be - raised""" + """If the server is not running but stopped an Exception should be raised.""" frontend = PanelFrontend(render_fn=Mock()) with pytest.raises(RuntimeError, match="Server is not running."): frontend.stop_server() @@ -48,8 +47,8 @@ def test_streamlit_frontend_start_stop_server(subprocess_mock): assert call_args[0] == sys.executable assert call_args[1].exists() assert str(call_args[1]).endswith("panel_serve_render_fn.py") - assert len(call_args)==2 - + assert len(call_args) == 2 + assert env_variables["LIGHTNING_FLOW_NAME"] == "root.my.flow" assert env_variables["LIGHTNING_RENDER_ADDRESS"] == "hostname" assert env_variables["LIGHTNING_RENDER_FUNCTION"] == "_noop_render_fn" @@ -66,6 +65,7 @@ def test_streamlit_frontend_start_stop_server(subprocess_mock): # Then subprocess_mock.Popen().kill.assert_called_once() + def _call_me(state): assert isinstance(state, AppState) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 9135995814a78..360d46549bdc2 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -10,7 +10,7 @@ @pytest.fixture(autouse=True, scope="module") def mock_settings_env_vars(): - """Set the LIGHTNING environment variables""" + """Set the LIGHTNING environment variables.""" with mock.patch.dict( os.environ, { @@ -30,7 +30,7 @@ def do_nothing(_): @pytest.fixture(autouse=True, scope="module") def mock_request_state(): - """Avoid requests to the api""" + """Avoid requests to the api.""" with mock.patch( "lightning_app.frontend.utilities.app_state_watcher.AppStateWatcher._start_watching", do_nothing, @@ -39,20 +39,20 @@ def mock_request_state(): def render_fn(app): - """Test function that just passes through the app""" + """Test function that just passes through the app.""" return app def test_view(): - """We have a helper _view function that provides the AppStateWatcher as argument to render_fn - and returns the result""" + """We have a helper _view function that provides the AppStateWatcher as argument to render_fn and returns the + result.""" result = _view() assert isinstance(result, AppStateWatcher) @mock.patch("panel.serve") def test_serve(pn_serve: mock.MagicMock): - """We can run python panel_serve_render_fn to serve the render_fn""" + """We can run python panel_serve_render_fn to serve the render_fn.""" _serve() pn_serve.assert_called_once_with( {"root.lit_panel": _view}, address="localhost", port=61896, websocket_origin="*", show=False diff --git a/tests/tests_app/frontend/utilities/conftest.py b/tests/tests_app/frontend/utilities/conftest.py index 48d903699ed07..915855f9e9b6a 100644 --- a/tests/tests_app/frontend/utilities/conftest.py +++ b/tests/tests_app/frontend/utilities/conftest.py @@ -1,9 +1,10 @@ +import os from re import L -import pytest from unittest import mock -import os -FLOW_SUB= "lit_panel" +import pytest + +FLOW_SUB = "lit_panel" FLOW = f"root.{FLOW_SUB}" PORT = 61896 @@ -31,23 +32,27 @@ "app_state": {"stage": "running"}, } + def _request_state(self): _state = APP_STATE self._store_state(_state) + def render_fn(app): - """Test function that just passes through the app""" + """Test function that just passes through the app.""" return app + @pytest.fixture(autouse=True, scope="module") def mock_request_state(): - """Avoid requests to the api""" + """Avoid requests to the api.""" with mock.patch("lightning_app.utilities.state.AppState._request_state", _request_state): yield + @pytest.fixture(autouse=True, scope="module") def mock_settings_env_vars(): - """Set the LIGHTNING environment variables""" + """Set the LIGHTNING environment variables.""" with mock.patch.dict( os.environ, { @@ -60,22 +65,23 @@ def mock_settings_env_vars(): ): yield + def do_nothing(): """Be lazy!""" @pytest.fixture(autouse=True, scope="module") def mock_start_websocket(): - """Avoid starting the websocket""" - with mock.patch( - "lightning_app.frontend.utilities.app_state_comm._start_websocket", do_nothing - ): + """Avoid starting the websocket.""" + with mock.patch("lightning_app.frontend.utilities.app_state_comm._start_websocket", do_nothing): yield + @pytest.fixture def app_state_state(): return APP_STATE.copy() + @pytest.fixture def flow_state_state(): - return FLOW_STATE.copy() \ No newline at end of file + return FLOW_STATE.copy() diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index cccc453d13310..4176b332544b3 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -1,14 +1,9 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app -state changes""" +"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" import os from unittest import mock from lightning_app.core.constants import APP_SERVER_PORT -from lightning_app.frontend.utilities.app_state_comm import ( - _get_ws_url, - _run_callbacks, - watch_app_state, -) +from lightning_app.frontend.utilities.app_state_comm import _get_ws_url, _run_callbacks, watch_app_state def do_nothing(): @@ -16,25 +11,25 @@ def do_nothing(): def test_get_ws_url_when_local(): - """The websocket uses port APP_SERVER_PORT when local""" + """The websocket uses port APP_SERVER_PORT when local.""" assert _get_ws_url() == f"ws://localhost:{APP_SERVER_PORT}/api/v1/ws" @mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "some_url"}) def test_get_ws_url_when_cloud(): - """The websocket uses port 8080 when LIGHTNING_APP_STATE_URL is set""" + """The websocket uses port 8080 when LIGHTNING_APP_STATE_URL is set.""" assert _get_ws_url() == "ws://localhost:8080/api/v1/ws" def test_watch_app_state(): - """We can watch the app state and run a callback function when it changes""" + """We can watch the app state and run a callback function when it changes.""" callback = mock.MagicMock() # When watch_app_state(callback) - + # Here we would like to send messages via the web socket # For testing the web socket is not started. See conftest.py # So we need to manually trigger _run_callbacks here _run_callbacks() # Then - callback.assert_called_once() \ No newline at end of file + callback.assert_called_once() diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index f4f5b173b7397..62a4061e73c71 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,4 +1,4 @@ -"""The AppStateWatcher enables a Frontend to +"""The AppStateWatcher enables a Frontend to. - subscribe to app state changes - to access and change the app state. @@ -9,8 +9,9 @@ from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher from lightning_app.utilities.state import AppState + def test_init(flow_state_state: dict): - """We can instantiate the AppStateWatcher + """We can instantiate the AppStateWatcher. - the .state is set - the .state is scoped to the flow state @@ -21,7 +22,7 @@ def test_init(flow_state_state: dict): def test_handle_state_changed(flow_state_state: dict): - """We can handle state changes by updating the state""" + """We can handle state changes by updating the state.""" app = AppStateWatcher() org_state = app.state app._handle_state_changed() @@ -30,7 +31,7 @@ def test_handle_state_changed(flow_state_state: dict): def test_is_singleton(): - """The AppStateWatcher is a singleton for efficiency reasons""" + """The AppStateWatcher is a singleton for efficiency reasons.""" watcher1 = AppStateWatcher() watcher2 = AppStateWatcher() assert watcher1 is watcher2 diff --git a/tests/tests_app/frontend/utilities/test_other.py b/tests/tests_app/frontend/utilities/test_other.py index 79f2c947803ba..68d74185c39a6 100644 --- a/tests/tests_app/frontend/utilities/test_other.py +++ b/tests/tests_app/frontend/utilities/test_other.py @@ -1,4 +1,4 @@ -"""We have some utility functions that can be used across frontends""" +"""We have some utility functions that can be used across frontends.""" import inspect import os @@ -11,16 +11,16 @@ def test_get_flow_state(flow_state_state: dict): - """We have a method to get an AppState scoped to the Flow state""" + """We have a method to get an AppState scoped to the Flow state.""" # When flow_state = get_flow_state() # Then assert isinstance(flow_state, AppState) - assert flow_state._state == flow_state_state # pylint: disable=protected-access + assert flow_state._state == flow_state_state # pylint: disable=protected-access def test_get_render_fn_from_environment(): - """We have a method to get the render_fn from the environment""" + """We have a method to get the render_fn from the environment.""" # When render_fn = get_render_fn_from_environment() # Then @@ -33,11 +33,9 @@ def some_fn(_): def test__get_frontend_environment(): - """We have a utility function to get the frontend render_fn environment""" + """We have a utility function to get the frontend render_fn environment.""" # When - env = get_frontend_environment( - flow="root.lit_frontend", render_fn=some_fn, host="myhost", port=1234 - ) + env = get_frontend_environment(flow="root.lit_frontend", render_fn=some_fn, host="myhost", port=1234) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" From 78a152e67ee913fd6a985d8077ed4078b8a26561 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sat, 9 Jul 2022 13:22:06 +0200 Subject: [PATCH 008/103] add panel as a requirement --- requirements/app/ui.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/app/ui.txt b/requirements/app/ui.txt index 28df7f9c2ffe0..b5fa36b782ebe 100644 --- a/requirements/app/ui.txt +++ b/requirements/app/ui.txt @@ -1 +1,2 @@ streamlit>=1.3.1 +panel From 8974b1d2809bcbd324df3e8f4549d863cfb02415 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sat, 9 Jul 2022 13:25:35 +0200 Subject: [PATCH 009/103] fix flake errors --- src/lightning_app/frontend/utilities/other.py | 2 +- tests/tests_app/frontend/utilities/conftest.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py index fdfb8ef6ec40f..254ddbe247644 100644 --- a/src/lightning_app/frontend/utilities/other.py +++ b/src/lightning_app/frontend/utilities/other.py @@ -6,7 +6,7 @@ import inspect import os import pydoc -from typing import Callable, Union +from typing import Callable from lightning_app.core.flow import LightningFlow from lightning_app.utilities.state import AppState diff --git a/tests/tests_app/frontend/utilities/conftest.py b/tests/tests_app/frontend/utilities/conftest.py index 915855f9e9b6a..067e7eede34f0 100644 --- a/tests/tests_app/frontend/utilities/conftest.py +++ b/tests/tests_app/frontend/utilities/conftest.py @@ -1,5 +1,4 @@ import os -from re import L from unittest import mock import pytest From e4e8593635b0fc1bd93455342bfe2d8f929daa64 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sat, 9 Jul 2022 19:26:30 +0200 Subject: [PATCH 010/103] work around that I cannot run lightning app when developing --- .../add_web_ui/panel/examples/__init__.py | 0 .../add_web_ui/panel/examples/app_basic.py | 4 +- .../examples/app_interact_from_component.py | 4 +- .../examples/app_interact_from_frontend.py | 6 +- .../panel/examples/app_state_comm.py | 82 ++++++++++++++++++ .../panel/examples/app_state_watcher.py | 86 +++++++++++++++++++ .../add_web_ui/panel/examples/copy.sh | 2 + .../add_web_ui/panel/examples/other.py | 64 ++++++++++++++ .../panel/examples/panel_frontend.py | 82 ++++++++++++++++++ .../panel/examples/panel_serve_render_fn.py | 56 ++++++++++++ 10 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/__init__.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/copy.sh create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/other.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/__init__.py b/docs/source-app/workflows/add_web_ui/panel/examples/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py index 7f769bd1300bb..dea482806e598 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py @@ -2,7 +2,9 @@ import panel as pn import lightning as L -from lightning_app.frontend.panel import PanelFrontend +# Todo: change import +# from lightning_app.frontend.panel import PanelFrontend +from panel_frontend import PanelFrontend def your_panel_app(app): diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py index ecc8d833b14ce..22b85f7cb0494 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py @@ -4,7 +4,9 @@ import panel as pn import lightning as L -from lightning_app.frontend.panel import AppStateWatcher, PanelFrontend +# Todo: change import +from panel_frontend import PanelFrontend +from app_state_watcher import AppStateWatcher pn.extension(sizing_mode="stretch_width") diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py index 4f37fc53dc2bd..1e1ac1886f5d3 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py @@ -2,11 +2,11 @@ import panel as pn import lightning as L -from lightning_app.frontend.panel import AppStateWatcher, PanelFrontend - +# Todo: Change import +from panel_frontend import PanelFrontend +from app_state_watcher import AppStateWatcher pn.extension(sizing_mode="stretch_width") - def your_panel_app(app: AppStateWatcher): submit_button = pn.widgets.Button(name="submit") diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py new file mode 100644 index 0000000000000..d407b2c1d58f2 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py @@ -0,0 +1,82 @@ +"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" +# Todo: Refactor with Streamlit +# Note: It would be nice one day to just watch changes within the Flow scope instead of whole app +from __future__ import annotations + +import asyncio +import logging +import os +import threading +from typing import Callable + +import websockets + +from lightning_app.core.constants import APP_SERVER_PORT + +_logger = logging.getLogger(__name__) + +_CALLBACKS = [] +_THREAD: None | threading.Thread = None + + +def _get_ws_port(): + if "LIGHTNING_APP_STATE_URL" in os.environ: + return 8080 + return APP_SERVER_PORT + + +def _get_ws_url(): + port = _get_ws_port() + return f"ws://localhost:{port}/api/v1/ws" + + +def _run_callbacks(): + for callback in _CALLBACKS: + callback() + + +def _target_fn(): + async def update_fn(): + ws_url = _get_ws_url() + _logger.debug("connecting to web socket %s", ws_url) + async with websockets.connect(ws_url) as websocket: # pylint: disable=no-member + while True: + await websocket.recv() + # Note: I have not seen use cases where the two lines below are needed + # Note: Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # while (time.time() - last_updated) < 0.2: + # time.sleep(0.05) + _logger.debug("App State Changed. Running callbacks") + _run_callbacks() + + asyncio.run(update_fn()) + + +def _start_websocket(): + global _THREAD # pylint: disable=global-statement + if not _THREAD: + _logger.debug("starting thread") + _THREAD = threading.Thread(target=_target_fn) + _THREAD.setDaemon(True) + _THREAD.start() + _logger.debug("thread started") + + +def watch_app_state(callback: Callable): + """Start the process that serves the UI at the given hostname and port number. + + Arguments: + callback: A function to run when the app state changes. Must be thread safe. + + Example: + + .. code-block:: python + + def handle_state_change(): + print("The App State Changed") + watch_app_state(handle_state_change) + """ + + _CALLBACKS.append(callback) + + _start_websocket() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py new file mode 100644 index 0000000000000..72fc6b524689b --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py @@ -0,0 +1,86 @@ +"""The AppStateWatcher enables a Frontend to. + +- subscribe to app state changes +- to access and change the app state. + +This is particularly useful for the PanelFrontend but can be used by other Frontends too. +""" +from __future__ import annotations + +import logging + +import param + +from app_state_comm import watch_app_state +from other import get_flow_state +from lightning_app.utilities.imports import requires +from lightning_app.utilities.state import AppState + +_logger = logging.getLogger(__name__) + +class AppStateWatcher(param.Parameterized): + """The AppStateWatcher enables a Frontend to. + + - subscribe to app state changes + - to access and change the app state. + + This is particularly useful for the PanelFrontend, but can be used by + other Frontends too. + + Example: + + .. code-block:: python + + import param + + app = AppStateWatcher() + + app.state.counter = 1 + + + @param.depends(app.param.state, watch=True) + def update(state): + print(f"The counter was updated to {state.counter}") + + + app.state.counter += 1 + + This would print 'The counter was updated to 2'. + + The AppStateWatcher is build on top of Param which is a framework like dataclass, attrs and + Pydantic which additionally provides powerful and unique features for building reactive apps. + """ + + state: AppState = param.ClassSelector( + class_=AppState, + doc=""" + The AppState holds the state of the app reduced to the scope of the Flow""", + ) + def __new__(cls): + # This makes the AppStateWatcher a *singleton*. + # The AppStateWatcher is a singleton to minimize the number of requests etc.. + if not hasattr(cls, "_instance"): + cls._instance = super().__new__(cls) + return cls._instance + + @requires("param") + def __init__(self): + if not hasattr(self, "_initilized"): + super().__init__(name="singleton") + self._start_watching() + self._initilized=True + + def _start_watching(self): + watch_app_state(self._handle_state_changed) + self._request_state() + + def _get_flow_state(self): + return get_flow_state() + + def _request_state(self): + self.state = self._get_flow_state() + _logger.debug("Request app state") + + def _handle_state_changed(self): + _logger.debug("Handle app state changed") + self._request_state() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh b/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh new file mode 100644 index 0000000000000..7bb011e5faf28 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh @@ -0,0 +1,2 @@ +cp -r ../../../../../../src/lightning_app/frontend/panel/*.py . +cp -r ../../../../../../src/lightning_app/frontend/utilities/*.py . \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/other.py b/docs/source-app/workflows/add_web_ui/panel/examples/other.py new file mode 100644 index 0000000000000..254ddbe247644 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/other.py @@ -0,0 +1,64 @@ +"""Utility functions for lightning Frontends.""" +# Todo: Refactor stream_lit and streamlit_base to use this functionality + +from __future__ import annotations + +import inspect +import os +import pydoc +from typing import Callable + +from lightning_app.core.flow import LightningFlow +from lightning_app.utilities.state import AppState + + +def get_render_fn_from_environment() -> Callable: + """Returns the render_fn function to serve in the Frontend.""" + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + module = pydoc.importfile(render_fn_module_file) + return getattr(module, render_fn_name) + + +def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppState: + """Returns a new AppState with the scope reduced to the given flow.""" + flow_name = flow.name if isinstance(flow, LightningFlow) else flow + flow_name_parts = flow_name.split(".")[1:] # exclude root + flow_state = state + for part in flow_name_parts: + flow_state = getattr(flow_state, part) + return flow_state + + +def get_flow_state() -> AppState: + """Returns an AppState scoped to the current Flow. + + Returns: + AppState: An AppState scoped to the current Flow. + """ + app_state = AppState() + app_state._request_state() # pylint: disable=protected-access + flow = os.environ["LIGHTNING_FLOW_NAME"] + flow_state = _reduce_to_flow_scope(app_state, flow) + return flow_state + + +def get_frontend_environment(flow: str, render_fn: Callable, port: int, host: str) -> os._Environ: + """Returns an _Environ with the environment variables for serving a Frontend app set. + + Args: + flow (str): The name of the flow, for example root.lit_frontend + render_fn (Callable): A function to render + port (int): The port number, for example 54321 + host (str): The host, for example 'localhost' + + Returns: + os._Environ: An environement + """ + env = os.environ.copy() + env["LIGHTNING_FLOW_NAME"] = flow + env["LIGHTNING_RENDER_FUNCTION"] = render_fn.__name__ + env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn).__file__ + env["LIGHTNING_RENDER_PORT"] = str(port) + env["LIGHTNING_RENDER_ADDRESS"] = str(host) + return env diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py new file mode 100644 index 0000000000000..93f5ffcb12297 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -0,0 +1,82 @@ +"""The PanelFrontend wraps your Panel code in your LightningFlow.""" +from __future__ import annotations + +import inspect +import logging +import pathlib +import subprocess +import sys +from typing import Callable + +from lightning_app.frontend.frontend import Frontend +from other import get_frontend_environment +from lightning_app.utilities.imports import requires +from lightning_app.utilities.log import get_frontend_logfile + +_logger = logging.getLogger("PanelFrontend") + + +class PanelFrontend(Frontend): + """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. + + To use this frontend, you must first install the `panel` package: + + .. code-block:: bash + + pip install panel + + Please note the Panel server will be logging output to error.log and output.log files + respectively. + + # Todo: Add Example + + Args: + render_fn: A pure function that contains your Panel code. This function must accept + exactly one argument, the `AppStateWatcher` object which you can use to get and + set variables in your flow (see example below). This function must return a + Panel Viewable. + + Raises: + TypeError: Raised if the render_fn is a class method + """ + + @requires("panel") + def __init__(self, render_fn: Callable): + # Todo: enable the render_fn to be a .py or .ipynb file + # Todo: enable the render_fn to not accept an AppStateWatcher as argument + super().__init__() + + if inspect.ismethod(render_fn): + raise TypeError( + "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a " "pure function." + ) + + self.render_fn = render_fn + self._process: None | subprocess.Popen = None + _logger.debug("initialized") + + def start_server(self, host: str, port: int) -> None: + _logger.debug("starting server %s %s", host, port) + env = get_frontend_environment( + self.flow.name, + self.render_fn, + port, + host, + ) + std_err_out = get_frontend_logfile("error.log") + std_out_out = get_frontend_logfile("output.log") + with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: + self._process = subprocess.Popen( # pylint: disable=consider-using-with + [ + sys.executable, + pathlib.Path(__file__).parent / "panel_serve_render_fn.py", + ], + env=env, + # stdout=stdout, + # stderr=stderr, + ) + + def stop_server(self) -> None: + if self._process is None: + raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") + self._process.kill() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py new file mode 100644 index 0000000000000..88d4f77c1fb63 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py @@ -0,0 +1,56 @@ +"""This file gets run by Python to lunch a Panel Server with Lightning. + +From here, we will call the render_fn that the user provided to the PanelFrontend. + +It requires the below environment variables to be set + +- LIGHTNING_FLOW_NAME +- LIGHTNING_RENDER_ADDRESS +- LIGHTNING_RENDER_FUNCTION +- LIGHTNING_RENDER_MODULE_FILE +- LIGHTNING_RENDER_PORT + +Example: + +.. code-block:: bash + + python panel_serve_render_fn +""" +from __future__ import annotations + +import logging +import os + +import panel as pn + +from app_state_watcher import AppStateWatcher +from other import get_render_fn_from_environment + +_logger = logging.getLogger(__name__) +_logger.setLevel(logging.DEBUG) + +def _view(): + render_fn = get_render_fn_from_environment() + app = AppStateWatcher() + print("_view", app) + return render_fn(app) + + +def _get_websocket_origin() -> str: + # Todo: Improve this. I don't know how to find the specific host(s). + # I tried but it did not work in cloud + return "*" + + +def _serve(): + port = int(os.environ["LIGHTNING_RENDER_PORT"]) + address = os.environ["LIGHTNING_RENDER_ADDRESS"] + url = os.environ["LIGHTNING_FLOW_NAME"] + websocket_origin = _get_websocket_origin() + + pn.serve({url: _view}, address=address, port=port, websocket_origin=websocket_origin, show=False) + _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) + + +if __name__ == "__main__": + _serve() From 8e5b62c5daf398384e6da59ef614f037d700cf5b Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sat, 9 Jul 2022 19:26:43 +0200 Subject: [PATCH 011/103] fix singleton issues --- .../frontend/utilities/app_state_watcher.py | 16 ++++++---- .../utilities/test_app_state_watcher.py | 29 +++++++++++++++---- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 35ad8926f83a0..79c1f116f9f79 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -50,6 +50,8 @@ def update(state): The AppStateWatcher is build on top of Param which is a framework like dataclass, attrs and Pydantic which additionally provides powerful and unique features for building reactive apps. + + Please note the AppStateWatcher is a singleton, i.e. only one instance is instantiated """ state: AppState = param.ClassSelector( @@ -61,14 +63,18 @@ def update(state): def __new__(cls): # This makes the AppStateWatcher a *singleton*. # The AppStateWatcher is a singleton to minimize the number of requests etc.. - if not hasattr(cls, "instance"): - cls.instance = super().__new__(cls) - return cls.instance + if not hasattr(cls, "_instance"): + cls._instance = super().__new__(cls) + return cls._instance @requires("param") def __init__(self): - super().__init__() - self._start_watching() + # Its critical to initialize only once + # See https://github.com/holoviz/param/issues/643 + if not hasattr(self, "_initilized"): + super().__init__() + self._start_watching() + self._initilized = True def _start_watching(self): watch_app_state(self._handle_state_changed) diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index 62a4061e73c71..e879f549fc326 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -22,7 +22,10 @@ def test_init(flow_state_state: dict): def test_handle_state_changed(flow_state_state: dict): - """We can handle state changes by updating the state.""" + """We can handle state changes by updating the state. + + - the .state is scoped to the flow state + """ app = AppStateWatcher() org_state = app.state app._handle_state_changed() @@ -31,7 +34,23 @@ def test_handle_state_changed(flow_state_state: dict): def test_is_singleton(): - """The AppStateWatcher is a singleton for efficiency reasons.""" - watcher1 = AppStateWatcher() - watcher2 = AppStateWatcher() - assert watcher1 is watcher2 + """The AppStateWatcher is a singleton for efficiency reasons. + + Its key that __new__ and __init__ of AppStateWatcher is only called once. + See https://github.com/holoviz/param/issues/643 + """ + # When + app1 = AppStateWatcher() + name1 = app1.name + state1 = app1.state + + app2 = AppStateWatcher() + name2 = app2.name + state2 = app2.state + + # Then + assert app1 is app2 + assert name1 == name2 + assert app1.name == name2 + assert state1 is state2 + assert app1.state is state2 From 9ca50852ce2d47d8e0965c08f262c721ddb44114 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sun, 10 Jul 2022 04:23:04 +0200 Subject: [PATCH 012/103] fix test issues --- .gitignore | 3 + .../panel/examples/app_state_watcher.py | 15 +- .../examples/app_streamlit_github_render.py | 275 ++++++++++++++++++ .../panel/examples/panel_serve_render_fn.py | 7 +- .../frontend/panel/panel_serve_render_fn.py | 4 +- .../frontend/utilities/app_state_watcher.py | 14 +- tests/tests_app/frontend/panel/conftest.py | 54 ++++ .../panel/test_panel_serve_render_fn.py | 21 +- .../tests_app/frontend/utilities/conftest.py | 8 +- .../utilities/test_app_state_watcher.py | 4 +- 10 files changed, 369 insertions(+), 36 deletions(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py create mode 100644 tests/tests_app/frontend/panel/conftest.py diff --git a/.gitignore b/.gitignore index 556905e208e7c..ea65e03fd12a4 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,6 @@ tags # Lightning StreamlitFrontend or PanelFrontend .storage + +# Personal scripts +script.* diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py index 72fc6b524689b..fdfc165b1daff 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py @@ -49,13 +49,16 @@ def update(state): The AppStateWatcher is build on top of Param which is a framework like dataclass, attrs and Pydantic which additionally provides powerful and unique features for building reactive apps. + + Please note the AppStateWatcher is a singleton, i.e. only one instance is instantiated """ state: AppState = param.ClassSelector( - class_=AppState, + class_=AppState, constant=True, doc=""" The AppState holds the state of the app reduced to the scope of the Flow""", ) + def __new__(cls): # This makes the AppStateWatcher a *singleton*. # The AppStateWatcher is a singleton to minimize the number of requests etc.. @@ -65,10 +68,13 @@ def __new__(cls): @requires("param") def __init__(self): + # Its critical to initialize only once + # See https://github.com/holoviz/param/issues/643 if not hasattr(self, "_initilized"): - super().__init__(name="singleton") + super().__init__() self._start_watching() - self._initilized=True + self.param.state.allow_None=False + self._initilized = True def _start_watching(self): watch_app_state(self._handle_state_changed) @@ -78,7 +84,8 @@ def _get_flow_state(self): return get_flow_state() def _request_state(self): - self.state = self._get_flow_state() + with param.edit_constant(self): + self.state = self._get_flow_state() _logger.debug("Request app state") def _handle_state_changed(self): diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py new file mode 100644 index 0000000000000..4098eeb7cd509 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py @@ -0,0 +1,275 @@ +import io +import os +import subprocess +import sys +from copy import deepcopy +from functools import partial +from subprocess import Popen +from typing import Dict, List, Optional + +from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow +from lightning.app import structures +from lightning.app.components.python import TracerPythonScript +from lightning.app.frontend import StreamlitFrontend +from lightning.app.storage.path import Path +from lightning.app.utilities.state import AppState + + +class GithubRepoRunner(TracerPythonScript): + def __init__( + self, + id: str, + github_repo: str, + script_path: str, + script_args: List[str], + requirements: List[str], + cloud_compute: Optional[CloudCompute] = None, + **kwargs, + ): + """The GithubRepoRunner Component clones a repo, + runs a specific script with provided arguments and collect logs. + + Arguments: + id: Identified of the component. + github_repo: The Github Repo URL to clone. + script_path: The path to the script to execute. + script_args: The arguments to be provided to the script. + requirements: The python requirements tp run the script. + cloud_compute: The object to select the cloud instance. + """ + super().__init__( + script_path=__file__, + script_args=script_args, + cloud_compute=cloud_compute, + cloud_build_config=BuildConfig(requirements=requirements), + ) + self.script_path=script_path + self.id = id + self.github_repo = github_repo + self.kwargs = kwargs + self.logs = [] + + def run(self, *args, **kwargs): + # 1. Hack: Patch stdout so we can capture the logs. + string_io = io.StringIO() + sys.stdout = string_io + + # 2: Use git command line to clone the repo. + repo_name = self.github_repo.split("/")[-1].replace(".git", "") + cwd = os.path.dirname(__file__) + subprocess.Popen( + f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() + + # 3: Execute the parent run method of the TracerPythonScript class. + os.chdir(os.path.join(cwd, repo_name)) + super().run(*args, **kwargs) + + # 4: Get all the collected logs and add them to the state. + # This isn't optimal as heavy, but works for this demo purpose. + self.logs = string_io.getvalue() + string_io.close() + + def configure_layout(self): + return {"name": self.id, "content": self} + + +class PyTorchLightningGithubRepoRunner(GithubRepoRunner): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.best_model_path = None + self.best_model_score = None + + def configure_tracer(self): + from pytorch_lightning import Trainer + from pytorch_lightning.callbacks import Callback + + tracer = super().configure_tracer() + + class TensorboardServerLauncher(Callback): + def __init__(self, work): + # The provided `work` is the + # current ``PyTorchLightningScript`` work. + self.w = work + + def on_train_start(self, trainer, *_): + # Add `host` and `port` for tensorboard to work in the cloud. + cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" + server_args = f"--host {self.w.host} --port {self.w.port}" + Popen(cmd + " " + server_args, shell=True) + + def trainer_pre_fn(self, *args, work=None, **kwargs): + # Intercept Trainer __init__ call + # and inject a ``TensorboardServerLauncher`` component. + kwargs["callbacks"].append(TensorboardServerLauncher(work)) + return {}, args, kwargs + + # 5. Patch the `__init__` method of the Trainer + # to inject our callback with a reference to the work. + tracer.add_traced( + Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) + return tracer + + def on_after_run(self, end_script_globals): + import torch + # 1. Once the script has finished to execute, + # we can collect its globals and access any objects. + trainer = end_script_globals["cli"].trainer + checkpoint_callback = trainer.checkpoint_callback + lightning_module = trainer.lightning_module + + # 2. From the checkpoint_callback, + # we are accessing the best model weights + checkpoint = torch.load(checkpoint_callback.best_model_path) + + # 3. Load the best weights and torchscript the model. + lightning_module.load_state_dict(checkpoint["state_dict"]) + lightning_module.to_torchscript(f"{self.name}.pt") + + # 4. Use lightning.app.storage.Pathto create a reference to the + # torch scripted model. In the cloud with multiple machines, + # by simply passing this reference to another work, + # it triggers automatically a file transfer. + self.best_model_path = Path(f"{self.name}.pt") + + # 5. Keep track of the metrics. + self.best_model_score = float(checkpoint_callback.best_model_score) + + +class KerasGithubRepoRunner(GithubRepoRunner): + """Left to the users to implement""" + +class TensorflowGithubRepoRunner(GithubRepoRunner): + """Left to the users to implement""" + +GITHUB_REPO_RUNNERS = { + "PyTorch Lightning": PyTorchLightningGithubRepoRunner, + "Keras": KerasGithubRepoRunner, + "Tensorflow": TensorflowGithubRepoRunner, +} + + +class Flow(LightningFlow): + def __init__(self): + super().__init__() + # 1: Keep track of the requests within the state + self.requests = [] + # 2: Create a dictionary of components. + self.ws = structures.Dict() + + def run(self): + # Iterate continuously over all requests + for request_id, request in enumerate(self.requests): + self._handle_request(request_id, deepcopy(request)) + + def _handle_request(self, request_id: int, request: Dict): + # 1: Create a name and find selected framework + name = f"w_{request_id}" + ml_framework = request["train"].pop("ml_framework") + + # 2: If the component hasn't been created yet, create it. + if name not in self.ws: + work_cls = GITHUB_REPO_RUNNERS[ml_framework] + work = work_cls(id=request["id"], **request["train"]) + self.ws[name] = work + + # 3: Run the component + self.ws[name].run() + + # 4: Once the component has finished, add metadata to the request. + if self.ws[name].best_model_path: + request = self.requests[request_id] + request["best_model_score"] = self.ws[name].best_model_score + request["best_model_path"] = self.ws[name].best_model_path + + def configure_layout(self): + # Create a StreamLit UI for the user to run his Github Repo. + return StreamlitFrontend(render_fn=render_fn) + + +def render_fn(state: AppState): + import streamlit as st + + def page_create_new_run(): + st.markdown("# Create a new Run 🎈") + id = st.text_input("Name your run", value="my_first_run") + github_repo = st.text_input( + "Enter a Github Repo URL", value="https://github.com/Lightning-AI/lightning-quick-start.git" + ) + + default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" + default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" + + script_path = st.text_input("Enter your script to run", value="train_script.py") + script_args = st.text_input("Enter your base script arguments", value=default_script_args) + requirements = st.text_input("Enter your requirements", value=default_requirements) + ml_framework = st.radio( + "Select your ML Training Frameworks", options=["PyTorch Lightning", "Keras", "Tensorflow"] + ) + + if ml_framework not in ("PyTorch Lightning"): + st.write(f"{ml_framework} isn't supported yet.") + return + + clicked = st.button("Submit") + if clicked: + new_request = { + "id": id, + "train": { + "github_repo": github_repo, + "script_path": script_path, + "script_args": script_args.split(" "), + "requirements": requirements.split(" "), + "ml_framework": ml_framework, + }, + } + state.requests = state.requests + [new_request] + + def page_view_run_lists(): + st.markdown("# Run Lists 🎈") + for idx, request in enumerate(state.requests): + work = state._state["structures"]["ws"]["works"][f"w_{idx}"] + with st.expander(f"Expand to view Run {idx}", expanded=False): + if st.checkbox(f"Expand to view your configuration", key=str(idx)): + st.json(request) + if st.checkbox(f"Expand to view logs", key=str(idx)): + st.code(body=work["vars"]["logs"]) + if st.checkbox(f"Expand to view your work state", key=str(idx)): + work["vars"].pop("logs") + st.json(work) + best_model_score = request.get("best_model_score", None) + if best_model_score: + if st.checkbox(f"Expand to view your run performance", key=str(idx)): + st.json( + {"best_model_score": best_model_score, "best_model_path": request.get("best_model_path")} + ) + + def page_view_app_state(): + st.markdown("# App State 🎈") + st.write(state._state) + + page_names_to_funcs = { + "Create a new Run": page_create_new_run, + "View your Runs": page_view_run_lists, + "View the App state": page_view_app_state, + } + + selected_page = st.sidebar.selectbox("Select a page", page_names_to_funcs.keys()) + page_names_to_funcs[selected_page]() + + +class RootFlow(LightningFlow): + def __init__(self): + super().__init__() + self.flow = Flow() + + def run(self): + self.flow.run() + + def configure_layout(self): + selection_tab = [{"name": "Run your Github Repo", "content": self.flow}] + run_tabs = [e.configure_layout() for e in self.flow.ws.values()] + return selection_tab + run_tabs + + +app = LightningApp(RootFlow()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py index 88d4f77c1fb63..22090450d0ed8 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py @@ -32,13 +32,16 @@ def _view(): render_fn = get_render_fn_from_environment() app = AppStateWatcher() - print("_view", app) return render_fn(app) def _get_websocket_origin() -> str: - # Todo: Improve this. I don't know how to find the specific host(s). + # Todo: Improve this to remove WARNING + # I don't know how to find the specific host(s). # I tried but it did not work in cloud + # + # WARNING:bokeh.server.util:Host wildcard '*' will allow connections originating from multiple + # (or possibly all) hostnames or IPs. Use non-wildcard values to restrict access explicitly return "*" diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py index 64db99664247c..08f16f5756ab5 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -29,7 +29,7 @@ _logger = logging.getLogger(__name__) -def _view(): +def _view_fn(): render_fn = get_render_fn_from_environment() app = AppStateWatcher() return render_fn(app) @@ -47,7 +47,7 @@ def _serve(): url = os.environ["LIGHTNING_FLOW_NAME"] websocket_origin = _get_websocket_origin() - pn.serve({url: _view}, address=address, port=port, websocket_origin=websocket_origin, show=False) + pn.serve({url: _view_fn}, address=address, port=port, websocket_origin=websocket_origin, show=False) _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 79c1f116f9f79..26692b442ce57 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -55,7 +55,7 @@ def update(state): """ state: AppState = param.ClassSelector( - class_=AppState, + class_=AppState, constant=True, doc=""" The AppState holds the state of the app reduced to the scope of the Flow""", ) @@ -72,10 +72,17 @@ def __init__(self): # Its critical to initialize only once # See https://github.com/holoviz/param/issues/643 if not hasattr(self, "_initilized"): - super().__init__() + super().__init__(name="singleton") self._start_watching() + self.param.state.allow_None=False self._initilized = True + # The below was observed when using mocking during testing + if not self.state: + raise Exception(".state has not been set.") + if not self.state._state: + raise Exception(".state._state has not been set.") + def _start_watching(self): watch_app_state(self._handle_state_changed) self._request_state() @@ -84,7 +91,8 @@ def _get_flow_state(self): return get_flow_state() def _request_state(self): - self.state = self._get_flow_state() + with param.edit_constant(self): + self.state = self._get_flow_state() _logger.debug("Request app state") def _handle_state_changed(self): diff --git a/tests/tests_app/frontend/panel/conftest.py b/tests/tests_app/frontend/panel/conftest.py new file mode 100644 index 0000000000000..d2cbb5b97e046 --- /dev/null +++ b/tests/tests_app/frontend/panel/conftest.py @@ -0,0 +1,54 @@ +import os +from unittest import mock + +import pytest + +FLOW_SUB = "lit_panel" +FLOW = f"root.{FLOW_SUB}" +PORT = 61896 + +FLOW_STATE = { + "vars": { + "_paths": {}, + "_layout": {"target": f"http://localhost:{PORT}/{FLOW}"}, + }, + "calls": {}, + "flows": {}, + "works": {}, + "structures": {}, + "changes": {}, +} + +APP_STATE = { + "vars": {"_paths": {}, "_layout": [{"name": "home", "content": FLOW}]}, + "calls": {}, + "flows": { + FLOW_SUB: FLOW_STATE, + }, + "works": {}, + "structures": {}, + "changes": {}, + "app_state": {"stage": "running"}, +} + + +def _request_state(self): + _state = APP_STATE + self._store_state(_state) + + +@pytest.fixture(autouse=True, scope="module") +def mock_request_state(): + """Avoid requests to the api.""" + with mock.patch("lightning_app.utilities.state.AppState._request_state", _request_state): + yield + +def do_nothing(): + """Be lazy!""" + + +@pytest.fixture(autouse=True, scope="module") +def mock_start_websocket(): + """Avoid starting the websocket.""" + with mock.patch("lightning_app.frontend.utilities.app_state_comm._start_websocket", do_nothing): + yield \ No newline at end of file diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 360d46549bdc2..b5ffbd7e00b03 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -4,7 +4,7 @@ import pytest -from lightning_app.frontend.panel.panel_serve_render_fn import _serve, _view +from lightning_app.frontend.panel.panel_serve_render_fn import _serve, _view_fn from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher @@ -23,21 +23,6 @@ def mock_settings_env_vars(): ): yield - -def do_nothing(_): - """Be lazy!""" - - -@pytest.fixture(autouse=True, scope="module") -def mock_request_state(): - """Avoid requests to the api.""" - with mock.patch( - "lightning_app.frontend.utilities.app_state_watcher.AppStateWatcher._start_watching", - do_nothing, - ): - yield - - def render_fn(app): """Test function that just passes through the app.""" return app @@ -46,7 +31,7 @@ def render_fn(app): def test_view(): """We have a helper _view function that provides the AppStateWatcher as argument to render_fn and returns the result.""" - result = _view() + result = _view_fn() assert isinstance(result, AppStateWatcher) @@ -55,5 +40,5 @@ def test_serve(pn_serve: mock.MagicMock): """We can run python panel_serve_render_fn to serve the render_fn.""" _serve() pn_serve.assert_called_once_with( - {"root.lit_panel": _view}, address="localhost", port=61896, websocket_origin="*", show=False + {"root.lit_panel": _view_fn}, address="localhost", port=61896, websocket_origin="*", show=False ) diff --git a/tests/tests_app/frontend/utilities/conftest.py b/tests/tests_app/frontend/utilities/conftest.py index 067e7eede34f0..0254bbcbde012 100644 --- a/tests/tests_app/frontend/utilities/conftest.py +++ b/tests/tests_app/frontend/utilities/conftest.py @@ -37,11 +37,6 @@ def _request_state(self): self._store_state(_state) -def render_fn(app): - """Test function that just passes through the app.""" - return app - - @pytest.fixture(autouse=True, scope="module") def mock_request_state(): """Avoid requests to the api.""" @@ -64,6 +59,9 @@ def mock_settings_env_vars(): ): yield +def render_fn(app): + """Test function that just passes through the app.""" + return app def do_nothing(): """Be lazy!""" diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index e879f549fc326..2938e373f3aa7 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,6 +1,6 @@ -"""The AppStateWatcher enables a Frontend to. +"""The AppStateWatcher enables a Frontend to -- subscribe to app state changes +- subscribe to app state changes. - to access and change the app state. This is particularly useful for the PanelFrontend but can be used by other Frontends too. From 9295e09a6b584775451f45ba6f4cef10025826db Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sun, 10 Jul 2022 08:48:20 +0200 Subject: [PATCH 013/103] refactor and add support for serving files --- src/lightning_app/frontend/panel/__init__.py | 2 + .../frontend/panel/panel_frontend.py | 25 ++-- ...fn.py => panel_serve_render_fn_or_file.py} | 37 +++++- .../frontend/utilities/app_state_comm.py | 3 +- .../frontend/utilities/app_state_watcher.py | 21 ++- src/lightning_app/frontend/utilities/other.py | 20 +-- .../frontend/{utilities => }/conftest.py | 45 ++++--- tests/tests_app/frontend/panel/app_panel.py | 4 + tests/tests_app/frontend/panel/conftest.py | 54 -------- .../frontend/panel/test_panel_frontend.py | 34 +++-- .../panel/test_panel_serve_render_file.py | 47 +++++++ .../panel/test_panel_serve_render_fn.py | 123 +++++++++++------- .../frontend/utilities/test_app_state_comm.py | 13 +- .../utilities/test_app_state_watcher.py | 37 +++++- .../frontend/utilities/test_other.py | 34 +++-- 15 files changed, 315 insertions(+), 184 deletions(-) rename src/lightning_app/frontend/panel/{panel_serve_render_fn.py => panel_serve_render_fn_or_file.py} (56%) rename tests/tests_app/frontend/{utilities => }/conftest.py (65%) create mode 100644 tests/tests_app/frontend/panel/app_panel.py delete mode 100644 tests/tests_app/frontend/panel/conftest.py create mode 100644 tests/tests_app/frontend/panel/test_panel_serve_render_file.py diff --git a/src/lightning_app/frontend/panel/__init__.py b/src/lightning_app/frontend/panel/__init__.py index 59b26d49ebacd..e09fc92430697 100644 --- a/src/lightning_app/frontend/panel/__init__.py +++ b/src/lightning_app/frontend/panel/__init__.py @@ -1,3 +1,5 @@ +"""The PanelFrontend and AppStateWatcher makes it easy to create lightning apps +with the Panel data app framework""" from lightning_app.frontend.panel.panel_frontend import PanelFrontend from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 9b5874e42d5a1..8ccaa94caa897 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -31,27 +31,26 @@ class PanelFrontend(Frontend): # Todo: Add Example Args: - render_fn: A pure function that contains your Panel code. This function must accept - exactly one argument, the `AppStateWatcher` object which you can use to get and - set variables in your flow (see example below). This function must return a - Panel Viewable. + render_fn_or_file: A pure function or the path to a .py or .ipynb file. + The function must be a pure function that contains your Panel code. + The function can optionally accept an `AppStateWatcher` argument. + The function must return a Panel Viewable. Raises: - TypeError: Raised if the render_fn is a class method + TypeError: Raised if the render_fn_or_file is a class method """ @requires("panel") - def __init__(self, render_fn: Callable): - # Todo: enable the render_fn to be a .py or .ipynb file - # Todo: enable the render_fn to not accept an AppStateWatcher as argument + def __init__(self, render_fn_or_file: Callable | str): super().__init__() - if inspect.ismethod(render_fn): + if inspect.ismethod(render_fn_or_file): raise TypeError( - "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a " "pure function." + "The `PanelFrontend` doesn't support `render_fn_or_file` being a method. " + "Please, use a pure function." ) - self.render_fn = render_fn + self.render_fn_or_file = render_fn_or_file self._process: None | subprocess.Popen = None _logger.debug("initialized") @@ -59,7 +58,7 @@ def start_server(self, host: str, port: int) -> None: _logger.debug("starting server %s %s", host, port) env = get_frontend_environment( self.flow.name, - self.render_fn, + self.render_fn_or_file, port, host, ) @@ -69,7 +68,7 @@ def start_server(self, host: str, port: int) -> None: self._process = subprocess.Popen( # pylint: disable=consider-using-with [ sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn.py", + pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", ], env=env, stdout=stdout, diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py similarity index 56% rename from src/lightning_app/frontend/panel/panel_serve_render_fn.py rename to src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py index 08f16f5756ab5..23de1d47d8431 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py @@ -6,18 +6,26 @@ - LIGHTNING_FLOW_NAME - LIGHTNING_RENDER_ADDRESS -- LIGHTNING_RENDER_FUNCTION -- LIGHTNING_RENDER_MODULE_FILE - LIGHTNING_RENDER_PORT +As well as either + +- LIGHTNING_RENDER_FUNCTION + LIGHTNING_RENDER_MODULE_FILE + +or + +- LIGHTNING_RENDER_FILE + + Example: .. code-block:: bash - python panel_serve_render_fn + python panel_serve_render_fn_or_file """ from __future__ import annotations +import inspect import logging import os @@ -28,12 +36,21 @@ _logger = logging.getLogger(__name__) +def _get_render_fn(): + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + return get_render_fn_from_environment(render_fn_name, render_fn_module_file) -def _view_fn(): - render_fn = get_render_fn_from_environment() +def _render_fn_wrapper(): + render_fn = _get_render_fn() app = AppStateWatcher() return render_fn(app) +def _get_view_fn(): + render_fn = _get_render_fn() + if inspect.signature(render_fn).parameters: + return _render_fn_wrapper + return render_fn def _get_websocket_origin() -> str: # Todo: Improve this. I don't know how to find the specific host(s). @@ -41,13 +58,21 @@ def _get_websocket_origin() -> str: return "*" +def _get_view(): + if "LIGHTNING_RENDER_FILE" in os.environ: + return os.environ["LIGHTNING_RENDER_FILE"] + return _get_view_fn() + + def _serve(): port = int(os.environ["LIGHTNING_RENDER_PORT"]) address = os.environ["LIGHTNING_RENDER_ADDRESS"] url = os.environ["LIGHTNING_FLOW_NAME"] websocket_origin = _get_websocket_origin() - pn.serve({url: _view_fn}, address=address, port=port, websocket_origin=websocket_origin, show=False) + view = _get_view() + + pn.serve({url: view}, address=address, port=port, websocket_origin=websocket_origin, show=False) _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index d407b2c1d58f2..5643f3a26c6b8 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -1,4 +1,5 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" +"""The watch_app_state function enables us to trigger a callback function when ever the +app state changes.""" # Todo: Refactor with Streamlit # Note: It would be nice one day to just watch changes within the Flow scope instead of whole app from __future__ import annotations diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 26692b442ce57..4515e05ddc03c 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -8,6 +8,7 @@ from __future__ import annotations import logging +import os import param @@ -55,7 +56,8 @@ def update(state): """ state: AppState = param.ClassSelector( - class_=AppState, constant=True, + class_=AppState, + constant=True, doc=""" The AppState holds the state of the app reduced to the scope of the Flow""", ) @@ -74,7 +76,7 @@ def __init__(self): if not hasattr(self, "_initilized"): super().__init__(name="singleton") self._start_watching() - self.param.state.allow_None=False + self.param.state.allow_None = False self._initilized = True # The below was observed when using mocking during testing @@ -84,17 +86,14 @@ def __init__(self): raise Exception(".state._state has not been set.") def _start_watching(self): - watch_app_state(self._handle_state_changed) - self._request_state() + watch_app_state(self._update_flow_state) + self._update_flow_state() - def _get_flow_state(self): - return get_flow_state() + def _get_flow_state(self) -> AppState: + flow = os.environ["LIGHTNING_FLOW_NAME"] + return get_flow_state(flow) - def _request_state(self): + def _update_flow_state(self): with param.edit_constant(self): self.state = self._get_flow_state() _logger.debug("Request app state") - - def _handle_state_changed(self): - _logger.debug("Handle app state changed") - self._request_state() diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py index 254ddbe247644..a8b0e1b6c82cd 100644 --- a/src/lightning_app/frontend/utilities/other.py +++ b/src/lightning_app/frontend/utilities/other.py @@ -12,10 +12,8 @@ from lightning_app.utilities.state import AppState -def get_render_fn_from_environment() -> Callable: +def get_render_fn_from_environment(render_fn_name: str, render_fn_module_file: str) -> Callable: """Returns the render_fn function to serve in the Frontend.""" - render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] - render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] module = pydoc.importfile(render_fn_module_file) return getattr(module, render_fn_name) @@ -30,7 +28,7 @@ def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppStat return flow_state -def get_flow_state() -> AppState: +def get_flow_state(flow: str) -> AppState: """Returns an AppState scoped to the current Flow. Returns: @@ -38,12 +36,13 @@ def get_flow_state() -> AppState: """ app_state = AppState() app_state._request_state() # pylint: disable=protected-access - flow = os.environ["LIGHTNING_FLOW_NAME"] flow_state = _reduce_to_flow_scope(app_state, flow) return flow_state -def get_frontend_environment(flow: str, render_fn: Callable, port: int, host: str) -> os._Environ: +def get_frontend_environment( + flow: str, render_fn_or_file: Callable | str, port: int, host: str +) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. Args: @@ -57,8 +56,13 @@ def get_frontend_environment(flow: str, render_fn: Callable, port: int, host: st """ env = os.environ.copy() env["LIGHTNING_FLOW_NAME"] = flow - env["LIGHTNING_RENDER_FUNCTION"] = render_fn.__name__ - env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn).__file__ env["LIGHTNING_RENDER_PORT"] = str(port) env["LIGHTNING_RENDER_ADDRESS"] = str(host) + + if isinstance(render_fn_or_file, str): + env["LIGHTNING_RENDER_FILE"] = render_fn_or_file + else: + env["LIGHTNING_RENDER_FUNCTION"] = render_fn_or_file.__name__ + env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn_or_file).__file__ + return env diff --git a/tests/tests_app/frontend/utilities/conftest.py b/tests/tests_app/frontend/conftest.py similarity index 65% rename from tests/tests_app/frontend/utilities/conftest.py rename to tests/tests_app/frontend/conftest.py index 0254bbcbde012..6effc5e7b828e 100644 --- a/tests/tests_app/frontend/utilities/conftest.py +++ b/tests/tests_app/frontend/conftest.py @@ -1,9 +1,11 @@ +"""Test configuration""" +# pylint: disable=protected-access import os from unittest import mock import pytest -FLOW_SUB = "lit_panel" +FLOW_SUB = "lit_flow" FLOW = f"root.{FLOW_SUB}" PORT = 61896 @@ -36,6 +38,9 @@ def _request_state(self): _state = APP_STATE self._store_state(_state) +@pytest.fixture() +def flow(): + return FLOW @pytest.fixture(autouse=True, scope="module") def mock_request_state(): @@ -43,26 +48,6 @@ def mock_request_state(): with mock.patch("lightning_app.utilities.state.AppState._request_state", _request_state): yield - -@pytest.fixture(autouse=True, scope="module") -def mock_settings_env_vars(): - """Set the LIGHTNING environment variables.""" - with mock.patch.dict( - os.environ, - { - "LIGHTNING_FLOW_NAME": FLOW, - "LIGHTNING_RENDER_ADDRESS": "localhost", - "LIGHTNING_RENDER_FUNCTION": "render_fn", - "LIGHTNING_RENDER_MODULE_FILE": __file__, - "LIGHTNING_RENDER_PORT": f"{PORT}", - }, - ): - yield - -def render_fn(app): - """Test function that just passes through the app.""" - return app - def do_nothing(): """Be lazy!""" @@ -76,9 +61,27 @@ def mock_start_websocket(): @pytest.fixture def app_state_state(): + """Returns an AppState dict""" return APP_STATE.copy() @pytest.fixture def flow_state_state(): + """Returns an AppState dict scoped to the flow""" return FLOW_STATE.copy() + + +# @pytest.fixture() +# def mock_settings_env_vars_fn(): +# """Set the LIGHTNING environment variables.""" +# with mock.patch.dict( +# os.environ, +# { +# "LIGHTNING_FLOW_NAME": FLOW, +# "LIGHTNING_RENDER_ADDRESS": "localhost", +# "LIGHTNING_RENDER_FUNCTION": "render_fn", +# "LIGHTNING_RENDER_MODULE_FILE": __file__, +# "LIGHTNING_RENDER_PORT": f"{PORT}", +# }, +# ): +# yield \ No newline at end of file diff --git a/tests/tests_app/frontend/panel/app_panel.py b/tests/tests_app/frontend/panel/app_panel.py new file mode 100644 index 0000000000000..a0ef75f7f31d9 --- /dev/null +++ b/tests/tests_app/frontend/panel/app_panel.py @@ -0,0 +1,4 @@ +"""Test App""" +import panel as pn + +pn.pane.Markdown("# Panel App").servable() diff --git a/tests/tests_app/frontend/panel/conftest.py b/tests/tests_app/frontend/panel/conftest.py deleted file mode 100644 index d2cbb5b97e046..0000000000000 --- a/tests/tests_app/frontend/panel/conftest.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -from unittest import mock - -import pytest - -FLOW_SUB = "lit_panel" -FLOW = f"root.{FLOW_SUB}" -PORT = 61896 - -FLOW_STATE = { - "vars": { - "_paths": {}, - "_layout": {"target": f"http://localhost:{PORT}/{FLOW}"}, - }, - "calls": {}, - "flows": {}, - "works": {}, - "structures": {}, - "changes": {}, -} - -APP_STATE = { - "vars": {"_paths": {}, "_layout": [{"name": "home", "content": FLOW}]}, - "calls": {}, - "flows": { - FLOW_SUB: FLOW_STATE, - }, - "works": {}, - "structures": {}, - "changes": {}, - "app_state": {"stage": "running"}, -} - - -def _request_state(self): - _state = APP_STATE - self._store_state(_state) - - -@pytest.fixture(autouse=True, scope="module") -def mock_request_state(): - """Avoid requests to the api.""" - with mock.patch("lightning_app.utilities.state.AppState._request_state", _request_state): - yield - -def do_nothing(): - """Be lazy!""" - - -@pytest.fixture(autouse=True, scope="module") -def mock_start_websocket(): - """Avoid starting the websocket.""" - with mock.patch("lightning_app.frontend.utilities.app_state_comm._start_websocket", do_nothing): - yield \ No newline at end of file diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index c22f559875dbf..efd77f7ed5381 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -1,3 +1,5 @@ +"""The PanelFrontend wraps your Panel code in your LightningFlow.""" +# pylint: disable=protected-access, too-few-public-methods import os import runpy import sys @@ -13,7 +15,7 @@ def test_stop_server_not_running(): """If the server is not running but stopped an Exception should be raised.""" - frontend = PanelFrontend(render_fn=Mock()) + frontend = PanelFrontend(render_fn_or_file=Mock()) with pytest.raises(RuntimeError, match="Server is not running."): frontend.stop_server() @@ -23,19 +25,23 @@ def _noop_render_fn(_): class MockFlow(LightningFlow): + """Test Flow""" + @property def name(self): + """Return name""" return "root.my.flow" - def run(self): - pass + def run(self): # pylint: disable=arguments-differ + "Be lazy!" @mock.patch("lightning_app.frontend.panel.panel_frontend.subprocess") -def test_streamlit_frontend_start_stop_server(subprocess_mock): - """Test that `PanelFrontend.start_server()` invokes subprocess.Popen with the right parameters.""" +def test_panel_frontend_start_stop_server(subprocess_mock): + """Test that `PanelFrontend.start_server()` invokes subprocess.Popen with the right + parameters.""" # Given - frontend = PanelFrontend(render_fn=_noop_render_fn) + frontend = PanelFrontend(render_fn_or_file=_noop_render_fn) frontend.flow = MockFlow() # When frontend.start_server(host="hostname", port=1111) @@ -46,7 +52,7 @@ def test_streamlit_frontend_start_stop_server(subprocess_mock): call_args = subprocess_mock.method_calls[0].args[0] assert call_args[0] == sys.executable assert call_args[1].exists() - assert str(call_args[1]).endswith("panel_serve_render_fn.py") + assert str(call_args[1]).endswith("panel_serve_render_fn_or_file.py") assert len(call_args) == 2 assert env_variables["LIGHTNING_FLOW_NAME"] == "root.my.flow" @@ -80,15 +86,19 @@ def _call_me(state): "LIGHTNING_RENDER_PORT": "61896", }, ) -def test_panel_wrapper_calls_render_fn(*_): - runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn") +def test_panel_wrapper_calls_render_fn_or_file(*_): + """Run the panel_serve_render_fn_or_file""" + runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn_or_file") # TODO: find a way to assert that _call_me got called def test_method_exception(): - class A: - def render_fn(self): + """The PanelFrontend does not support render_fn_or_file being a method + and should raise an Exception""" + + class _DummyClass: + def _render_fn(self): pass with pytest.raises(TypeError, match="being a method"): - PanelFrontend(render_fn=A().render_fn) + PanelFrontend(render_fn_or_file=_DummyClass()._render_fn) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_file.py b/tests/tests_app/frontend/panel/test_panel_serve_render_file.py new file mode 100644 index 0000000000000..0b3e99394e7c6 --- /dev/null +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_file.py @@ -0,0 +1,47 @@ +"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with +Lightning.""" +# pylint: disable=redefined-outer-name +import os +import pathlib +from unittest import mock + +import pytest + +from lightning_app.frontend.panel.panel_serve_render_fn_or_file import _serve + + +@pytest.fixture(scope="module") +def render_file(): + """Returns the path to a Panel app file""" + path = pathlib.Path(__file__).parent / "app_panel.py" + path = path.relative_to(pathlib.Path.cwd()) + return str(path) + + +@pytest.fixture(autouse=True, scope="module") +def mock_settings_env_vars(render_file): + """Set the LIGHTNING environment variables.""" + + with mock.patch.dict( + os.environ, + { + "LIGHTNING_FLOW_NAME": "root.lit_flow", + "LIGHTNING_RENDER_ADDRESS": "localhost", + "LIGHTNING_RENDER_FILE": render_file, + "LIGHTNING_RENDER_PORT": "61896", + }, + ): + yield + + +@mock.patch("panel.serve") +def test_serve(pn_serve: mock.MagicMock, render_file): + """We can run python panel_serve_render_fn_or_file to serve the render_file.""" + _serve() + pn_serve.assert_called_once_with( + {"root.lit_flow": render_file}, + address="localhost", + port=61896, + websocket_origin="*", + show=False, + ) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index b5ffbd7e00b03..0085f8cb47053 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -1,44 +1,79 @@ -"""This panel_serve_render_fn file gets run by Python to lunch a Panel Server with Lightning.""" -import os -from unittest import mock - -import pytest - -from lightning_app.frontend.panel.panel_serve_render_fn import _serve, _view_fn -from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher - - -@pytest.fixture(autouse=True, scope="module") -def mock_settings_env_vars(): - """Set the LIGHTNING environment variables.""" - with mock.patch.dict( - os.environ, - { - "LIGHTNING_FLOW_NAME": "root.lit_panel", - "LIGHTNING_RENDER_ADDRESS": "localhost", - "LIGHTNING_RENDER_FUNCTION": "render_fn", - "LIGHTNING_RENDER_MODULE_FILE": __file__, - "LIGHTNING_RENDER_PORT": "61896", - }, - ): - yield - -def render_fn(app): - """Test function that just passes through the app.""" - return app - - -def test_view(): - """We have a helper _view function that provides the AppStateWatcher as argument to render_fn and returns the - result.""" - result = _view_fn() - assert isinstance(result, AppStateWatcher) - - -@mock.patch("panel.serve") -def test_serve(pn_serve: mock.MagicMock): - """We can run python panel_serve_render_fn to serve the render_fn.""" - _serve() - pn_serve.assert_called_once_with( - {"root.lit_panel": _view_fn}, address="localhost", port=61896, websocket_origin="*", show=False - ) +"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with +Lightning.""" +import os +from unittest import mock + +import pytest + +from lightning_app.frontend.panel.panel_serve_render_fn_or_file import ( + _serve, + _get_view_fn, + _render_fn_wrapper, +) +from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher + + +@pytest.fixture(autouse=True, scope="module") +def mock_settings_env_vars(): + """Set the LIGHTNING environment variables.""" + with mock.patch.dict( + os.environ, + { + "LIGHTNING_FLOW_NAME": "root.lit_flow", + "LIGHTNING_RENDER_ADDRESS": "localhost", + "LIGHTNING_RENDER_FUNCTION": "render_fn", + "LIGHTNING_RENDER_MODULE_FILE": __file__, + "LIGHTNING_RENDER_PORT": "61896", + }, + ): + yield + + +def render_fn(app): + """Test function that just passes through the app.""" + return app + + +def test_get_view_fn(): + """We have a helper get_view_fn function that create a function for our view. + + If the render_fn provides an argument an AppStateWatcher is provided as argument + """ + view_fn = _get_view_fn() + result = view_fn() + assert isinstance(result, AppStateWatcher) + + +def render_fn_no_args(): + """Test function with no arguments""" + return "Hello" + + +@mock.patch.dict( + os.environ, + { + "LIGHTNING_RENDER_FUNCTION": "render_fn_no_args", + "LIGHTNING_RENDER_MODULE_FILE": __file__, + }, +) +def test_get_view_fn_no_args(): + """We have a helper get_view_fn function that create a function for our view. + + If the render_fn provides an argument an AppStateWatcher is provided as argument + """ + view_fn = _get_view_fn() + result = view_fn() + assert result == "Hello" + + +@mock.patch("panel.serve") +def test_serve(pn_serve: mock.MagicMock): + """We can run python panel_serve_render_fn_or_file to serve the render_fn.""" + _serve() + pn_serve.assert_called_once_with( + {"root.lit_flow": _render_fn_wrapper}, + address="localhost", + port=61896, + websocket_origin="*", + show=False, + ) diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index 4176b332544b3..35d0368c3ffb3 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -1,9 +1,17 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" +"""The watch_app_state function enables us to trigger a callback function when +ever the app state changes.""" import os from unittest import mock from lightning_app.core.constants import APP_SERVER_PORT -from lightning_app.frontend.utilities.app_state_comm import _get_ws_url, _run_callbacks, watch_app_state +from lightning_app.frontend.utilities.app_state_comm import ( + _get_ws_url, + _run_callbacks, + watch_app_state, +) + +FLOW_SUB = "lit_flow" +FLOW = f"root.{FLOW_SUB}" def do_nothing(): @@ -21,6 +29,7 @@ def test_get_ws_url_when_cloud(): assert _get_ws_url() == "ws://localhost:8080/api/v1/ws" +@mock.patch.dict(os.environ, {"LIGHTNING_FLOW_NAME": "FLOW"}) def test_watch_app_state(): """We can watch the app state and run a callback function when it changes.""" callback = mock.MagicMock() diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index 2938e373f3aa7..cd1e64257f02f 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -6,9 +6,32 @@ This is particularly useful for the PanelFrontend but can be used by other Frontends too. """ # pylint: disable=protected-access +import os +from unittest import mock + +import pytest + from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher from lightning_app.utilities.state import AppState +FLOW_SUB = "lit_flow" +FLOW = f"root.{FLOW_SUB}" +PORT = 61896 + + +@pytest.fixture(autouse=True) +def mock_settings_env_vars(): + """Set the LIGHTNING environment variables.""" + with mock.patch.dict( + os.environ, + { + "LIGHTNING_FLOW_NAME": FLOW, + "LIGHTNING_RENDER_ADDRESS": "localhost", + "LIGHTNING_RENDER_PORT": f"{PORT}", + }, + ): + yield + def test_init(flow_state_state: dict): """We can instantiate the AppStateWatcher. @@ -16,19 +39,25 @@ def test_init(flow_state_state: dict): - the .state is set - the .state is scoped to the flow state """ + # When app = AppStateWatcher() + # Needed as AppStateWatcher is singleton and might have been + # instantiated and the state changed in other tests + app._update_flow_state() + + # Then assert isinstance(app.state, AppState) assert app.state._state == flow_state_state -def test_handle_state_changed(flow_state_state: dict): - """We can handle state changes by updating the state. - +def test_update_flow_state(flow_state_state: dict): + """We can update the state. + - the .state is scoped to the flow state """ app = AppStateWatcher() org_state = app.state - app._handle_state_changed() + app._update_flow_state() assert app.state is not org_state assert app.state._state == flow_state_state diff --git a/tests/tests_app/frontend/utilities/test_other.py b/tests/tests_app/frontend/utilities/test_other.py index 68d74185c39a6..da480ae0f8a41 100644 --- a/tests/tests_app/frontend/utilities/test_other.py +++ b/tests/tests_app/frontend/utilities/test_other.py @@ -1,6 +1,5 @@ """We have some utility functions that can be used across frontends.""" import inspect -import os from lightning_app.frontend.utilities.other import ( get_flow_state, @@ -10,35 +9,54 @@ from lightning_app.utilities.state import AppState -def test_get_flow_state(flow_state_state: dict): +def test_get_flow_state(flow_state_state: dict, flow): """We have a method to get an AppState scoped to the Flow state.""" # When - flow_state = get_flow_state() + flow_state = get_flow_state(flow) # Then assert isinstance(flow_state, AppState) assert flow_state._state == flow_state_state # pylint: disable=protected-access +def render_fn(): + """Do nothing""" + + def test_get_render_fn_from_environment(): """We have a method to get the render_fn from the environment.""" # When - render_fn = get_render_fn_from_environment() + result = get_render_fn_from_environment("render_fn", __file__) # Then - assert inspect.getfile(render_fn) == os.environ["LIGHTNING_RENDER_MODULE_FILE"] - assert render_fn.__name__ == os.environ["LIGHTNING_RENDER_FUNCTION"] + assert result.__name__ == render_fn.__name__ + assert inspect.getmodule(result).__file__ == __file__ def some_fn(_): """Be lazy!""" -def test__get_frontend_environment(): +def test_get_frontend_environment_fn(): """We have a utility function to get the frontend render_fn environment.""" # When - env = get_frontend_environment(flow="root.lit_frontend", render_fn=some_fn, host="myhost", port=1234) + env = get_frontend_environment( + flow="root.lit_frontend", render_fn_or_file=some_fn, host="myhost", port=1234 + ) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" assert env["LIGHTNING_RENDER_FUNCTION"] == "some_fn" assert env["LIGHTNING_RENDER_MODULE_FILE"] == __file__ assert env["LIGHTNING_RENDER_PORT"] == "1234" + + +def test_get_frontend_environment_file(): + """We have a utility function to get the frontend render_fn environment.""" + # When + env = get_frontend_environment( + flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234 + ) + # Then + assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" + assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" + assert env["LIGHTNING_RENDER_FILE"] == "app_panel.py" + assert env["LIGHTNING_RENDER_PORT"] == "1234" From 7cbf4460d6a8a3e83789c5df24768db97986edb8 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sun, 10 Jul 2022 14:07:16 +0200 Subject: [PATCH 014/103] refactor and improve --- .../add_web_ui/panel/examples/app_basic.py | 64 +- .../panel/examples/app_basic_script.py | 28 + .../panel/examples/app_github_render.py | 203 +++++++ .../examples/app_interact_from_component.py | 2 +- .../examples/app_interact_from_frontend.py | 2 +- .../panel/examples/app_state_comm.py | 3 +- .../panel/examples/app_state_watcher.py | 30 +- .../examples/app_streamlit_github_render.py | 551 +++++++++--------- .../add_web_ui/panel/examples/other.py | 20 +- .../panel/examples/panel_frontend.py | 25 +- .../panel/examples/panel_github_render.py | 186 ++++++ .../add_web_ui/panel/examples/panel_script.py | 13 + .../panel/examples/panel_serve_render_fn.py | 59 -- .../examples/panel_serve_render_fn_or_file.py | 81 +++ .../add_web_ui/panel/examples/state.json | 1 + .../add_web_ui/panel/tips_and_tricks.rst | 9 + .../frontend/panel/panel_frontend.py | 7 +- .../panel/panel_serve_render_fn_or_file.py | 6 +- .../frontend/utilities/app_state_comm.py | 7 +- .../frontend/utilities/app_state_watcher.py | 4 +- 20 files changed, 896 insertions(+), 405 deletions(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/state.json create mode 100644 docs/source-app/workflows/add_web_ui/panel/tips_and_tricks.rst diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py index dea482806e598..872dfada30e23 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py @@ -1,32 +1,32 @@ -# app.py -import panel as pn - -import lightning as L -# Todo: change import -# from lightning_app.frontend.panel import PanelFrontend -from panel_frontend import PanelFrontend - - -def your_panel_app(app): - return pn.pane.Markdown("hello") - - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend(render_fn=your_panel_app) - - def configure_layout(self): - return self._frontend - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - -app = L.LightningApp(LitApp()) +# app.py +import panel as pn + +import lightning as L +# Todo: change import +# from lightning_app.frontend.panel import PanelFrontend +from panel_frontend import PanelFrontend + + +def your_panel_app(app): + return pn.pane.Markdown("hello") + + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend(your_panel_app) + + def configure_layout(self): + return self._frontend + + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + +app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py new file mode 100644 index 0000000000000..2d275cb12f728 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py @@ -0,0 +1,28 @@ +# app.py +import panel as pn + +import lightning as L +# Todo: change import +# from lightning_app.frontend.panel import PanelFrontend +from panel_frontend import PanelFrontend + + +class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("panel_script.py") + + def configure_layout(self): + return self._frontend + + +class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + +app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py new file mode 100644 index 0000000000000..4ed0729201b65 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py @@ -0,0 +1,203 @@ +import io +import os +import subprocess +import sys +from copy import deepcopy +from functools import partial +from subprocess import Popen +from typing import Dict, List, Optional + +from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow +from lightning.app import structures +from lightning.app.components.python import TracerPythonScript +from panel_frontend import PanelFrontend +from lightning.app.storage.path import Path +from app_state_watcher import AppStateWatcher + +class GithubRepoRunner(TracerPythonScript): + def __init__( + self, + id: str, + github_repo: str, + script_path: str, + script_args: List[str], + requirements: List[str], + cloud_compute: Optional[CloudCompute] = None, + **kwargs, + ): + """The GithubRepoRunner Component clones a repo, + runs a specific script with provided arguments and collect logs. + + Arguments: + id: Identified of the component. + github_repo: The Github Repo URL to clone. + script_path: The path to the script to execute. + script_args: The arguments to be provided to the script. + requirements: The python requirements tp run the script. + cloud_compute: The object to select the cloud instance. + """ + super().__init__( + script_path=__file__, + script_args=script_args, + cloud_compute=cloud_compute, + cloud_build_config=BuildConfig(requirements=requirements), + ) + self.script_path=script_path + self.id = id + self.github_repo = github_repo + self.kwargs = kwargs + self.logs = [] + + def run(self, *args, **kwargs): + # 1. Hack: Patch stdout so we can capture the logs. + string_io = io.StringIO() + sys.stdout = string_io + + # 2: Use git command line to clone the repo. + repo_name = self.github_repo.split("/")[-1].replace(".git", "") + cwd = os.path.dirname(__file__) + subprocess.Popen( + f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() + + # 3: Execute the parent run method of the TracerPythonScript class. + os.chdir(os.path.join(cwd, repo_name)) + super().run(*args, **kwargs) + + # 4: Get all the collected logs and add them to the state. + # This isn't optimal as heavy, but works for this demo purpose. + self.logs = string_io.getvalue() + string_io.close() + + def configure_layout(self): + return {"name": self.id, "content": self} + + +class PyTorchLightningGithubRepoRunner(GithubRepoRunner): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.best_model_path = None + self.best_model_score = None + + def configure_tracer(self): + from pytorch_lightning import Trainer + from pytorch_lightning.callbacks import Callback + + tracer = super().configure_tracer() + + class TensorboardServerLauncher(Callback): + def __init__(self, work): + # The provided `work` is the + # current ``PyTorchLightningScript`` work. + self.w = work + + def on_train_start(self, trainer, *_): + # Add `host` and `port` for tensorboard to work in the cloud. + cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" + server_args = f"--host {self.w.host} --port {self.w.port}" + Popen(cmd + " " + server_args, shell=True) + + def trainer_pre_fn(self, *args, work=None, **kwargs): + # Intercept Trainer __init__ call + # and inject a ``TensorboardServerLauncher`` component. + kwargs["callbacks"].append(TensorboardServerLauncher(work)) + return {}, args, kwargs + + # 5. Patch the `__init__` method of the Trainer + # to inject our callback with a reference to the work. + tracer.add_traced( + Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) + return tracer + + def on_after_run(self, end_script_globals): + import torch + # 1. Once the script has finished to execute, + # we can collect its globals and access any objects. + trainer = end_script_globals["cli"].trainer + checkpoint_callback = trainer.checkpoint_callback + lightning_module = trainer.lightning_module + + # 2. From the checkpoint_callback, + # we are accessing the best model weights + checkpoint = torch.load(checkpoint_callback.best_model_path) + + # 3. Load the best weights and torchscript the model. + lightning_module.load_state_dict(checkpoint["state_dict"]) + lightning_module.to_torchscript(f"{self.name}.pt") + + # 4. Use lightning.app.storage.Pathto create a reference to the + # torch scripted model. In the cloud with multiple machines, + # by simply passing this reference to another work, + # it triggers automatically a file transfer. + self.best_model_path = Path(f"{self.name}.pt") + + # 5. Keep track of the metrics. + self.best_model_score = float(checkpoint_callback.best_model_score) + + +class KerasGithubRepoRunner(GithubRepoRunner): + """Left to the users to implement""" + +class TensorflowGithubRepoRunner(GithubRepoRunner): + """Left to the users to implement""" + +GITHUB_REPO_RUNNERS = { + "PyTorch Lightning": PyTorchLightningGithubRepoRunner, + "Keras": KerasGithubRepoRunner, + "Tensorflow": TensorflowGithubRepoRunner, +} + + +class Flow(LightningFlow): + def __init__(self): + super().__init__() + # 1: Keep track of the requests within the state + self.requests = [] + # 2: Create a dictionary of components. + self.ws = structures.Dict() + + def run(self): + # Iterate continuously over all requests + for request_id, request in enumerate(self.requests): + self._handle_request(request_id, deepcopy(request)) + + def _handle_request(self, request_id: int, request: Dict): + # 1: Create a name and find selected framework + name = f"w_{request_id}" + ml_framework = request["train"].pop("ml_framework") + + # 2: If the component hasn't been created yet, create it. + if name not in self.ws: + work_cls = GITHUB_REPO_RUNNERS[ml_framework] + work = work_cls(id=request["id"], **request["train"]) + self.ws[name] = work + + # 3: Run the component + self.ws[name].run() + + # 4: Once the component has finished, add metadata to the request. + if self.ws[name].best_model_path: + request = self.requests[request_id] + request["best_model_score"] = self.ws[name].best_model_score + request["best_model_path"] = self.ws[name].best_model_path + + def configure_layout(self): + # Create a StreamLit UI for the user to run his Github Repo. + return PanelFrontend("panel_github_render.py") + + +class RootFlow(LightningFlow): + def __init__(self): + super().__init__() + self.flow = Flow() + + def run(self): + self.flow.run() + + def configure_layout(self): + selection_tab = [{"name": "Run your Github Repo", "content": self.flow}] + run_tabs = [e.configure_layout() for e in self.flow.ws.values()] + return selection_tab + run_tabs + + +app = LightningApp(RootFlow()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py index 22b85f7cb0494..330243e5cdf01 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py @@ -25,7 +25,7 @@ class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend(render_fn=your_panel_app) + self._frontend = PanelFrontend(your_panel_app) self._last_update = dt.datetime.now() self.last_update = self._last_update.isoformat() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py index 1e1ac1886f5d3..39391c9300f30 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py @@ -28,7 +28,7 @@ def current_count(_): class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend(render_fn=your_panel_app) + self._frontend = PanelFrontend(your_panel_app) self.count = 0 def configure_layout(self): diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py index d407b2c1d58f2..5643f3a26c6b8 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py @@ -1,4 +1,5 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" +"""The watch_app_state function enables us to trigger a callback function when ever the +app state changes.""" # Todo: Refactor with Streamlit # Note: It would be nice one day to just watch changes within the Flow scope instead of whole app from __future__ import annotations diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py index fdfc165b1daff..c587fd75b32db 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py @@ -8,6 +8,7 @@ from __future__ import annotations import logging +import os import param @@ -18,6 +19,7 @@ _logger = logging.getLogger(__name__) + class AppStateWatcher(param.Parameterized): """The AppStateWatcher enables a Frontend to. @@ -54,7 +56,8 @@ def update(state): """ state: AppState = param.ClassSelector( - class_=AppState, constant=True, + class_=AppState, + constant=True, doc=""" The AppState holds the state of the app reduced to the scope of the Flow""", ) @@ -71,23 +74,26 @@ def __init__(self): # Its critical to initialize only once # See https://github.com/holoviz/param/issues/643 if not hasattr(self, "_initilized"): - super().__init__() + super().__init__(name="singleton") self._start_watching() - self.param.state.allow_None=False + self.param.state.allow_None = False self._initilized = True + # The below was observed when using mocking during testing + if not self.state: + raise Exception(".state has not been set.") + if not self.state._state: + raise Exception(".state._state has not been set.") + def _start_watching(self): - watch_app_state(self._handle_state_changed) - self._request_state() + watch_app_state(self._update_flow_state) + self._update_flow_state() - def _get_flow_state(self): - return get_flow_state() + def _get_flow_state(self) -> AppState: + flow = os.environ["LIGHTNING_FLOW_NAME"] + return get_flow_state(flow) - def _request_state(self): + def _update_flow_state(self): with param.edit_constant(self): self.state = self._get_flow_state() _logger.debug("Request app state") - - def _handle_state_changed(self): - _logger.debug("Handle app state changed") - self._request_state() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py index 4098eeb7cd509..76ceddf343b29 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py @@ -1,275 +1,278 @@ -import io -import os -import subprocess -import sys -from copy import deepcopy -from functools import partial -from subprocess import Popen -from typing import Dict, List, Optional - -from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow -from lightning.app import structures -from lightning.app.components.python import TracerPythonScript -from lightning.app.frontend import StreamlitFrontend -from lightning.app.storage.path import Path -from lightning.app.utilities.state import AppState - - -class GithubRepoRunner(TracerPythonScript): - def __init__( - self, - id: str, - github_repo: str, - script_path: str, - script_args: List[str], - requirements: List[str], - cloud_compute: Optional[CloudCompute] = None, - **kwargs, - ): - """The GithubRepoRunner Component clones a repo, - runs a specific script with provided arguments and collect logs. - - Arguments: - id: Identified of the component. - github_repo: The Github Repo URL to clone. - script_path: The path to the script to execute. - script_args: The arguments to be provided to the script. - requirements: The python requirements tp run the script. - cloud_compute: The object to select the cloud instance. - """ - super().__init__( - script_path=__file__, - script_args=script_args, - cloud_compute=cloud_compute, - cloud_build_config=BuildConfig(requirements=requirements), - ) - self.script_path=script_path - self.id = id - self.github_repo = github_repo - self.kwargs = kwargs - self.logs = [] - - def run(self, *args, **kwargs): - # 1. Hack: Patch stdout so we can capture the logs. - string_io = io.StringIO() - sys.stdout = string_io - - # 2: Use git command line to clone the repo. - repo_name = self.github_repo.split("/")[-1].replace(".git", "") - cwd = os.path.dirname(__file__) - subprocess.Popen( - f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() - - # 3: Execute the parent run method of the TracerPythonScript class. - os.chdir(os.path.join(cwd, repo_name)) - super().run(*args, **kwargs) - - # 4: Get all the collected logs and add them to the state. - # This isn't optimal as heavy, but works for this demo purpose. - self.logs = string_io.getvalue() - string_io.close() - - def configure_layout(self): - return {"name": self.id, "content": self} - - -class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.best_model_path = None - self.best_model_score = None - - def configure_tracer(self): - from pytorch_lightning import Trainer - from pytorch_lightning.callbacks import Callback - - tracer = super().configure_tracer() - - class TensorboardServerLauncher(Callback): - def __init__(self, work): - # The provided `work` is the - # current ``PyTorchLightningScript`` work. - self.w = work - - def on_train_start(self, trainer, *_): - # Add `host` and `port` for tensorboard to work in the cloud. - cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" - server_args = f"--host {self.w.host} --port {self.w.port}" - Popen(cmd + " " + server_args, shell=True) - - def trainer_pre_fn(self, *args, work=None, **kwargs): - # Intercept Trainer __init__ call - # and inject a ``TensorboardServerLauncher`` component. - kwargs["callbacks"].append(TensorboardServerLauncher(work)) - return {}, args, kwargs - - # 5. Patch the `__init__` method of the Trainer - # to inject our callback with a reference to the work. - tracer.add_traced( - Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) - return tracer - - def on_after_run(self, end_script_globals): - import torch - # 1. Once the script has finished to execute, - # we can collect its globals and access any objects. - trainer = end_script_globals["cli"].trainer - checkpoint_callback = trainer.checkpoint_callback - lightning_module = trainer.lightning_module - - # 2. From the checkpoint_callback, - # we are accessing the best model weights - checkpoint = torch.load(checkpoint_callback.best_model_path) - - # 3. Load the best weights and torchscript the model. - lightning_module.load_state_dict(checkpoint["state_dict"]) - lightning_module.to_torchscript(f"{self.name}.pt") - - # 4. Use lightning.app.storage.Pathto create a reference to the - # torch scripted model. In the cloud with multiple machines, - # by simply passing this reference to another work, - # it triggers automatically a file transfer. - self.best_model_path = Path(f"{self.name}.pt") - - # 5. Keep track of the metrics. - self.best_model_score = float(checkpoint_callback.best_model_score) - - -class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement""" - -class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement""" - -GITHUB_REPO_RUNNERS = { - "PyTorch Lightning": PyTorchLightningGithubRepoRunner, - "Keras": KerasGithubRepoRunner, - "Tensorflow": TensorflowGithubRepoRunner, -} - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Keep track of the requests within the state - self.requests = [] - # 2: Create a dictionary of components. - self.ws = structures.Dict() - - def run(self): - # Iterate continuously over all requests - for request_id, request in enumerate(self.requests): - self._handle_request(request_id, deepcopy(request)) - - def _handle_request(self, request_id: int, request: Dict): - # 1: Create a name and find selected framework - name = f"w_{request_id}" - ml_framework = request["train"].pop("ml_framework") - - # 2: If the component hasn't been created yet, create it. - if name not in self.ws: - work_cls = GITHUB_REPO_RUNNERS[ml_framework] - work = work_cls(id=request["id"], **request["train"]) - self.ws[name] = work - - # 3: Run the component - self.ws[name].run() - - # 4: Once the component has finished, add metadata to the request. - if self.ws[name].best_model_path: - request = self.requests[request_id] - request["best_model_score"] = self.ws[name].best_model_score - request["best_model_path"] = self.ws[name].best_model_path - - def configure_layout(self): - # Create a StreamLit UI for the user to run his Github Repo. - return StreamlitFrontend(render_fn=render_fn) - - -def render_fn(state: AppState): - import streamlit as st - - def page_create_new_run(): - st.markdown("# Create a new Run 🎈") - id = st.text_input("Name your run", value="my_first_run") - github_repo = st.text_input( - "Enter a Github Repo URL", value="https://github.com/Lightning-AI/lightning-quick-start.git" - ) - - default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" - default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" - - script_path = st.text_input("Enter your script to run", value="train_script.py") - script_args = st.text_input("Enter your base script arguments", value=default_script_args) - requirements = st.text_input("Enter your requirements", value=default_requirements) - ml_framework = st.radio( - "Select your ML Training Frameworks", options=["PyTorch Lightning", "Keras", "Tensorflow"] - ) - - if ml_framework not in ("PyTorch Lightning"): - st.write(f"{ml_framework} isn't supported yet.") - return - - clicked = st.button("Submit") - if clicked: - new_request = { - "id": id, - "train": { - "github_repo": github_repo, - "script_path": script_path, - "script_args": script_args.split(" "), - "requirements": requirements.split(" "), - "ml_framework": ml_framework, - }, - } - state.requests = state.requests + [new_request] - - def page_view_run_lists(): - st.markdown("# Run Lists 🎈") - for idx, request in enumerate(state.requests): - work = state._state["structures"]["ws"]["works"][f"w_{idx}"] - with st.expander(f"Expand to view Run {idx}", expanded=False): - if st.checkbox(f"Expand to view your configuration", key=str(idx)): - st.json(request) - if st.checkbox(f"Expand to view logs", key=str(idx)): - st.code(body=work["vars"]["logs"]) - if st.checkbox(f"Expand to view your work state", key=str(idx)): - work["vars"].pop("logs") - st.json(work) - best_model_score = request.get("best_model_score", None) - if best_model_score: - if st.checkbox(f"Expand to view your run performance", key=str(idx)): - st.json( - {"best_model_score": best_model_score, "best_model_path": request.get("best_model_path")} - ) - - def page_view_app_state(): - st.markdown("# App State 🎈") - st.write(state._state) - - page_names_to_funcs = { - "Create a new Run": page_create_new_run, - "View your Runs": page_view_run_lists, - "View the App state": page_view_app_state, - } - - selected_page = st.sidebar.selectbox("Select a page", page_names_to_funcs.keys()) - page_names_to_funcs[selected_page]() - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.flow = Flow() - - def run(self): - self.flow.run() - - def configure_layout(self): - selection_tab = [{"name": "Run your Github Repo", "content": self.flow}] - run_tabs = [e.configure_layout() for e in self.flow.ws.values()] - return selection_tab + run_tabs - - +import io +import os +import subprocess +import sys +from copy import deepcopy +from functools import partial +from subprocess import Popen +from typing import Dict, List, Optional + +from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow +from lightning.app import structures +from lightning.app.components.python import TracerPythonScript +from lightning.app.frontend import StreamlitFrontend +from lightning.app.storage.path import Path +from lightning.app.utilities.state import AppState + + +class GithubRepoRunner(TracerPythonScript): + def __init__( + self, + id: str, + github_repo: str, + script_path: str, + script_args: List[str], + requirements: List[str], + cloud_compute: Optional[CloudCompute] = None, + **kwargs, + ): + """The GithubRepoRunner Component clones a repo, + runs a specific script with provided arguments and collect logs. + + Arguments: + id: Identified of the component. + github_repo: The Github Repo URL to clone. + script_path: The path to the script to execute. + script_args: The arguments to be provided to the script. + requirements: The python requirements tp run the script. + cloud_compute: The object to select the cloud instance. + """ + super().__init__( + script_path=__file__, + script_args=script_args, + cloud_compute=cloud_compute, + cloud_build_config=BuildConfig(requirements=requirements), + ) + self.script_path=script_path + self.id = id + self.github_repo = github_repo + self.kwargs = kwargs + self.logs = [] + + def run(self, *args, **kwargs): + # 1. Hack: Patch stdout so we can capture the logs. + string_io = io.StringIO() + sys.stdout = string_io + + # 2: Use git command line to clone the repo. + repo_name = self.github_repo.split("/")[-1].replace(".git", "") + cwd = os.path.dirname(__file__) + subprocess.Popen( + f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() + + # 3: Execute the parent run method of the TracerPythonScript class. + os.chdir(os.path.join(cwd, repo_name)) + super().run(*args, **kwargs) + + # 4: Get all the collected logs and add them to the state. + # This isn't optimal as heavy, but works for this demo purpose. + self.logs = string_io.getvalue() + string_io.close() + + def configure_layout(self): + return {"name": self.id, "content": self} + + +class PyTorchLightningGithubRepoRunner(GithubRepoRunner): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.best_model_path = None + self.best_model_score = None + + def configure_tracer(self): + from pytorch_lightning import Trainer + from pytorch_lightning.callbacks import Callback + + tracer = super().configure_tracer() + + class TensorboardServerLauncher(Callback): + def __init__(self, work): + # The provided `work` is the + # current ``PyTorchLightningScript`` work. + self.w = work + + def on_train_start(self, trainer, *_): + # Add `host` and `port` for tensorboard to work in the cloud. + cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" + server_args = f"--host {self.w.host} --port {self.w.port}" + Popen(cmd + " " + server_args, shell=True) + + def trainer_pre_fn(self, *args, work=None, **kwargs): + # Intercept Trainer __init__ call + # and inject a ``TensorboardServerLauncher`` component. + kwargs["callbacks"].append(TensorboardServerLauncher(work)) + return {}, args, kwargs + + # 5. Patch the `__init__` method of the Trainer + # to inject our callback with a reference to the work. + tracer.add_traced( + Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) + return tracer + + def on_after_run(self, end_script_globals): + import torch + # 1. Once the script has finished to execute, + # we can collect its globals and access any objects. + trainer = end_script_globals["cli"].trainer + checkpoint_callback = trainer.checkpoint_callback + lightning_module = trainer.lightning_module + + # 2. From the checkpoint_callback, + # we are accessing the best model weights + checkpoint = torch.load(checkpoint_callback.best_model_path) + + # 3. Load the best weights and torchscript the model. + lightning_module.load_state_dict(checkpoint["state_dict"]) + lightning_module.to_torchscript(f"{self.name}.pt") + + # 4. Use lightning.app.storage.Pathto create a reference to the + # torch scripted model. In the cloud with multiple machines, + # by simply passing this reference to another work, + # it triggers automatically a file transfer. + self.best_model_path = Path(f"{self.name}.pt") + + # 5. Keep track of the metrics. + self.best_model_score = float(checkpoint_callback.best_model_score) + + +class KerasGithubRepoRunner(GithubRepoRunner): + """Left to the users to implement""" + +class TensorflowGithubRepoRunner(GithubRepoRunner): + """Left to the users to implement""" + +GITHUB_REPO_RUNNERS = { + "PyTorch Lightning": PyTorchLightningGithubRepoRunner, + "Keras": KerasGithubRepoRunner, + "Tensorflow": TensorflowGithubRepoRunner, +} + + +class Flow(LightningFlow): + def __init__(self): + super().__init__() + # 1: Keep track of the requests within the state + self.requests = [] + # 2: Create a dictionary of components. + self.ws = structures.Dict() + + def run(self): + # Iterate continuously over all requests + for request_id, request in enumerate(self.requests): + self._handle_request(request_id, deepcopy(request)) + + def _handle_request(self, request_id: int, request: Dict): + # 1: Create a name and find selected framework + name = f"w_{request_id}" + ml_framework = request["train"].pop("ml_framework") + + # 2: If the component hasn't been created yet, create it. + if name not in self.ws: + work_cls = GITHUB_REPO_RUNNERS[ml_framework] + work = work_cls(id=request["id"], **request["train"]) + self.ws[name] = work + + # 3: Run the component + self.ws[name].run() + + # 4: Once the component has finished, add metadata to the request. + if self.ws[name].best_model_path: + request = self.requests[request_id] + request["best_model_score"] = self.ws[name].best_model_score + request["best_model_path"] = self.ws[name].best_model_path + + def configure_layout(self): + # Create a StreamLit UI for the user to run his Github Repo. + return StreamlitFrontend(render_fn=render_fn) + + +def render_fn(state: AppState): + import json + with open("state.json", "w") as fp: + json.dump(state._state,fp) + import streamlit as st + + def page_create_new_run(): + st.markdown("# Create a new Run 🎈") + id = st.text_input("Name your run", value="my_first_run") + github_repo = st.text_input( + "Enter a Github Repo URL", value="https://github.com/Lightning-AI/lightning-quick-start.git" + ) + + default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" + default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" + + script_path = st.text_input("Enter your script to run", value="train_script.py") + script_args = st.text_input("Enter your base script arguments", value=default_script_args) + requirements = st.text_input("Enter your requirements", value=default_requirements) + ml_framework = st.radio( + "Select your ML Training Frameworks", options=["PyTorch Lightning", "Keras", "Tensorflow"] + ) + + if ml_framework not in ("PyTorch Lightning"): + st.write(f"{ml_framework} isn't supported yet.") + return + + clicked = st.button("Submit") + if clicked: + new_request = { + "id": id, + "train": { + "github_repo": github_repo, + "script_path": script_path, + "script_args": script_args.split(" "), + "requirements": requirements.split(" "), + "ml_framework": ml_framework, + }, + } + state.requests = state.requests + [new_request] + + def page_view_run_lists(): + st.markdown("# Run Lists 🎈") + for idx, request in enumerate(state.requests): + work = state._state["structures"]["ws"]["works"][f"w_{idx}"] + with st.expander(f"Expand to view Run {idx}", expanded=False): + if st.checkbox(f"Expand to view your configuration", key=str(idx)): + st.json(request) + if st.checkbox(f"Expand to view logs", key=str(idx)): + st.code(body=work["vars"]["logs"]) + if st.checkbox(f"Expand to view your work state", key=str(idx)): + work["vars"].pop("logs") + st.json(work) + best_model_score = request.get("best_model_score", None) + if best_model_score: + if st.checkbox(f"Expand to view your run performance", key=str(idx)): + st.json( + {"best_model_score": best_model_score, "best_model_path": request.get("best_model_path")} + ) + + def page_view_app_state(): + st.markdown("# App State 🎈") + st.write(state._state) + + page_names_to_funcs = { + "Create a new Run": page_create_new_run, + "View your Runs": page_view_run_lists, + "View the App state": page_view_app_state, + } + + selected_page = st.sidebar.selectbox("Select a page", page_names_to_funcs.keys()) + page_names_to_funcs[selected_page]() + + +class RootFlow(LightningFlow): + def __init__(self): + super().__init__() + self.flow = Flow() + + def run(self): + self.flow.run() + + def configure_layout(self): + selection_tab = [{"name": "Run your Github Repo", "content": self.flow}] + run_tabs = [e.configure_layout() for e in self.flow.ws.values()] + return selection_tab + run_tabs + + app = LightningApp(RootFlow()) \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/other.py b/docs/source-app/workflows/add_web_ui/panel/examples/other.py index 254ddbe247644..a8b0e1b6c82cd 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/other.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/other.py @@ -12,10 +12,8 @@ from lightning_app.utilities.state import AppState -def get_render_fn_from_environment() -> Callable: +def get_render_fn_from_environment(render_fn_name: str, render_fn_module_file: str) -> Callable: """Returns the render_fn function to serve in the Frontend.""" - render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] - render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] module = pydoc.importfile(render_fn_module_file) return getattr(module, render_fn_name) @@ -30,7 +28,7 @@ def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppStat return flow_state -def get_flow_state() -> AppState: +def get_flow_state(flow: str) -> AppState: """Returns an AppState scoped to the current Flow. Returns: @@ -38,12 +36,13 @@ def get_flow_state() -> AppState: """ app_state = AppState() app_state._request_state() # pylint: disable=protected-access - flow = os.environ["LIGHTNING_FLOW_NAME"] flow_state = _reduce_to_flow_scope(app_state, flow) return flow_state -def get_frontend_environment(flow: str, render_fn: Callable, port: int, host: str) -> os._Environ: +def get_frontend_environment( + flow: str, render_fn_or_file: Callable | str, port: int, host: str +) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. Args: @@ -57,8 +56,13 @@ def get_frontend_environment(flow: str, render_fn: Callable, port: int, host: st """ env = os.environ.copy() env["LIGHTNING_FLOW_NAME"] = flow - env["LIGHTNING_RENDER_FUNCTION"] = render_fn.__name__ - env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn).__file__ env["LIGHTNING_RENDER_PORT"] = str(port) env["LIGHTNING_RENDER_ADDRESS"] = str(host) + + if isinstance(render_fn_or_file, str): + env["LIGHTNING_RENDER_FILE"] = render_fn_or_file + else: + env["LIGHTNING_RENDER_FUNCTION"] = render_fn_or_file.__name__ + env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn_or_file).__file__ + return env diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py index 93f5ffcb12297..659940310e00c 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -31,27 +31,26 @@ class PanelFrontend(Frontend): # Todo: Add Example Args: - render_fn: A pure function that contains your Panel code. This function must accept - exactly one argument, the `AppStateWatcher` object which you can use to get and - set variables in your flow (see example below). This function must return a - Panel Viewable. + render_fn_or_file: A pure function or the path to a .py or .ipynb file. + The function must be a pure function that contains your Panel code. + The function can optionally accept an `AppStateWatcher` argument. + The function must return a Panel Viewable. Raises: - TypeError: Raised if the render_fn is a class method + TypeError: Raised if the render_fn_or_file is a class method """ @requires("panel") - def __init__(self, render_fn: Callable): - # Todo: enable the render_fn to be a .py or .ipynb file - # Todo: enable the render_fn to not accept an AppStateWatcher as argument + def __init__(self, render_fn_or_file: Callable | str): super().__init__() - if inspect.ismethod(render_fn): + if inspect.ismethod(render_fn_or_file): raise TypeError( - "The `PanelFrontend` doesn't support `render_fn` being a method. Please, use a " "pure function." + "The `PanelFrontend` doesn't support `render_fn_or_file` being a method. " + "Please, use a pure function." ) - self.render_fn = render_fn + self.render_fn_or_file = render_fn_or_file self._process: None | subprocess.Popen = None _logger.debug("initialized") @@ -59,7 +58,7 @@ def start_server(self, host: str, port: int) -> None: _logger.debug("starting server %s %s", host, port) env = get_frontend_environment( self.flow.name, - self.render_fn, + self.render_fn_or_file, port, host, ) @@ -69,7 +68,7 @@ def start_server(self, host: str, port: int) -> None: self._process = subprocess.Popen( # pylint: disable=consider-using-with [ sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn.py", + pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", ], env=env, # stdout=stdout, diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py new file mode 100644 index 0000000000000..af6e731bd6f1e --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py @@ -0,0 +1,186 @@ +import os +from unittest.mock import Mock + +import panel as pn +import param +from app_state_watcher import AppStateWatcher + +from lightning_app.utilities.state import AppState + +def to_str(value): + if isinstance(value, list): + return "\n".join([str(item) for item in value]) + return str(value) + +if "LIGHTNING_FLOW_NAME" in os.environ: + app = AppStateWatcher() +else: + class AppMock(param.Parameterized): + state = param.Parameter() + + app = AppMock(state=Mock()) + import json + + with open("state.json", "r") as fp: + app.state._state = json.load(fp) + app.state.requests = [ + { + "id": 0, + "train": { + "github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", + "script_path": "train_script.py", + "script_args": [ + "--trainer.max_epochs=5", + "--trainer.limit_train_batches=4", + "--trainer.limit_val_batches=4", + "--trainer.callbacks=ModelCheckpoint", + "--trainer.callbacks.monitor=val_acc", + ], + "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], + "ml_framework": "PyTorch Lightning", + }, + } + ] + + +ACCENT = "#792EE5" +LIGHTNING_SPINNER_URL = ( + "https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/spinners/material/" + "bar_chart_lightning_purple.svg" +) +LIGHTNING_SPINNER = pn.pane.HTML( + f"" +) +# Todo: Set JSON theme depending on template theme +# pn.pane.JSON.param.theme.default = "dark" +pn.pane.JSON.param.hover_preview.default = True + +pn.config.raw_css.append( + """ + .bk-root { + height: calc( 100vh - 200px ) !important; + } + """ +) +pn.extension("terminal", sizing_mode="stretch_width", template="fast") +pn.state.template.param.update(accent_base_color=ACCENT, header_background=ACCENT) + + +def create_new_page(): + title = "# Create a new run 🎈" + id_input = pn.widgets.TextInput(name="Name your run", value="my_first_run") + github_repo_input = pn.widgets.TextInput( + name="Enter a Github Repo URL", + value="https://github.com/Lightning-AI/lightning-quick-start.git", + ) + script_path_input = pn.widgets.TextInput( + name="Enter your script to run", value="train_script.py" + ) + + default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" + script_args_input = pn.widgets.TextInput( + name="Enter your base script arguments", value=default_script_args + ) + default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" + requirements_input = pn.widgets.TextInput( + name="Enter your requirements", value=default_requirements + ) + ml_framework_input = pn.widgets.RadioBoxGroup( + name="Select your ML Training Frameworks", + options=["PyTorch Lightning", "Keras", "Tensorflow"], + inline=True, + ) + submit_input = pn.widgets.Button(name="⚡ SUBMIT ⚡", button_type="primary") + + @pn.depends(submit_input, watch=True) + def create_new_run(_): + new_request = { + "id": id_input.value, + "train": { + "github_repo": github_repo_input.value, + "script_path": script_path_input.value, + "script_args": script_args_input.value.split(" "), + "requirements": requirements_input.value.split(" "), + "ml_framework": ml_framework_input.value, + }, + } + app.state.requests = app.state.requests + [new_request] + print("submitted", new_request) + + @pn.depends(ml_framework_input.param.value) + def message_or_button(ml_framework): + if ml_framework not in ("PyTorch Lightning"): + return f"{ml_framework} isn't supported yet." + else: + return submit_input + + return pn.Column( + title, + id_input, + github_repo_input, + script_path_input, + script_args_input, + requirements_input, + ml_framework_input, + message_or_button, + ) + + +def card_show_work(idx, request, state): + work = state["structures"]["ws"]["works"][f"w_{idx}"] + + def get_work_state(): + w = work["vars"].copy() + if "logs" in w: + w.pop("logs") + return pn.pane.JSON(w, theme="light", sizing_mode="stretch_both") + + options = { + "Expand to view your configuration": pn.pane.JSON( + request, theme="light", hover_preview=True, depth=4 + ), + "Expand to view logs": pn.Column( + pn.pane.Markdown( + "```bash\n" + to_str(work["vars"]["logs"]) + "\n```", + ), + height=800, + ), + "Expand to view your work state": get_work_state(), + } + selection_input = pn.widgets.RadioBoxGroup(name="Hello", options=list(options.keys())) + + @pn.depends(selection_input) + def selection_output(value): + return pn.panel(options[value], sizing_mode="stretch_both") + + return pn.Column( + selection_input, selection_output, sizing_mode="stretch_both", name=f"Run: {idx}" + ) + + +@pn.depends(app.param.state) +def view_run_lists_page(state: AppState): + title = "# Run Lists 🎈" + # Todo: Consider other layout than accordion. Don't think its that great + layout = pn.Accordion(sizing_mode="stretch_both") + print(state._request_state) + for idx, request in enumerate(state.requests): + layout.append(card_show_work(idx, request, state._state)) + layout.append(card_show_work(idx, request, state._state)) + return pn.Column(title, layout) + + +@pn.depends(app.param.state) +def app_state_page(state: AppState): + title = "# App State 🎈" + # Todo: Make this on stretch full heigh of its parent containe + json_output = pn.pane.JSON(state._state, theme="light", depth=6, max_height=800) + return pn.Column(title, json_output, scroll=True) + + +pn.Tabs( + ("New Run", create_new_page), + ("View your Runs", view_run_lists_page), + ("App State", app_state_page), + sizing_mode="stretch_both", +).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py new file mode 100644 index 0000000000000..b27678746d38f --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py @@ -0,0 +1,13 @@ +import os + +import panel as pn + +pn.extension(sizing_mode="stretch_width") + +pn.panel("# Hello Panel 4").servable() + +from app_state_watcher import AppStateWatcher + +app = AppStateWatcher() +pn.panel(os.environ.get("PANEL_AUTORELOAD", "no")).servable() +pn.pane.JSON(app.state._state, theme="light", height=300, width=500, depth=3).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py deleted file mode 100644 index 22090450d0ed8..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py +++ /dev/null @@ -1,59 +0,0 @@ -"""This file gets run by Python to lunch a Panel Server with Lightning. - -From here, we will call the render_fn that the user provided to the PanelFrontend. - -It requires the below environment variables to be set - -- LIGHTNING_FLOW_NAME -- LIGHTNING_RENDER_ADDRESS -- LIGHTNING_RENDER_FUNCTION -- LIGHTNING_RENDER_MODULE_FILE -- LIGHTNING_RENDER_PORT - -Example: - -.. code-block:: bash - - python panel_serve_render_fn -""" -from __future__ import annotations - -import logging -import os - -import panel as pn - -from app_state_watcher import AppStateWatcher -from other import get_render_fn_from_environment - -_logger = logging.getLogger(__name__) -_logger.setLevel(logging.DEBUG) - -def _view(): - render_fn = get_render_fn_from_environment() - app = AppStateWatcher() - return render_fn(app) - - -def _get_websocket_origin() -> str: - # Todo: Improve this to remove WARNING - # I don't know how to find the specific host(s). - # I tried but it did not work in cloud - # - # WARNING:bokeh.server.util:Host wildcard '*' will allow connections originating from multiple - # (or possibly all) hostnames or IPs. Use non-wildcard values to restrict access explicitly - return "*" - - -def _serve(): - port = int(os.environ["LIGHTNING_RENDER_PORT"]) - address = os.environ["LIGHTNING_RENDER_ADDRESS"] - url = os.environ["LIGHTNING_FLOW_NAME"] - websocket_origin = _get_websocket_origin() - - pn.serve({url: _view}, address=address, port=port, websocket_origin=websocket_origin, show=False) - _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) - - -if __name__ == "__main__": - _serve() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py new file mode 100644 index 0000000000000..d60b9ab9437b0 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py @@ -0,0 +1,81 @@ +"""This file gets run by Python to lunch a Panel Server with Lightning. + +From here, we will call the render_fn that the user provided to the PanelFrontend. + +It requires the below environment variables to be set + +- LIGHTNING_FLOW_NAME +- LIGHTNING_RENDER_ADDRESS +- LIGHTNING_RENDER_PORT + +As well as either + +- LIGHTNING_RENDER_FUNCTION + LIGHTNING_RENDER_MODULE_FILE + +or + +- LIGHTNING_RENDER_FILE + + +Example: + +.. code-block:: bash + + python panel_serve_render_fn_or_file +""" +from __future__ import annotations + +import inspect +import logging +import os + +import panel as pn + +from app_state_watcher import AppStateWatcher +from other import get_render_fn_from_environment + +_logger = logging.getLogger(__name__) + +def _get_render_fn(): + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + return get_render_fn_from_environment(render_fn_name, render_fn_module_file) + +def _render_fn_wrapper(): + render_fn = _get_render_fn() + app = AppStateWatcher() + return render_fn(app) + +def _get_view_fn(): + render_fn = _get_render_fn() + if inspect.signature(render_fn).parameters: + return _render_fn_wrapper + return render_fn + +def _get_websocket_origin() -> str: + # Todo: Improve this. I don't know how to find the specific host(s). + # I tried but it did not work in cloud + return "*" + + +def _get_view(): + if "LIGHTNING_RENDER_FILE" in os.environ: + return os.environ["LIGHTNING_RENDER_FILE"] + return _get_view_fn() + + +def _serve(): + port = int(os.environ["LIGHTNING_RENDER_PORT"]) + address = os.environ["LIGHTNING_RENDER_ADDRESS"] + url = os.environ["LIGHTNING_FLOW_NAME"] + websocket_origin = _get_websocket_origin() + autoreload = os.environ.get("PANEL_AUTORELOAD", "no")=="yes" + + view = _get_view() + + pn.serve({url: view}, address=address, port=port, websocket_origin=websocket_origin, show=False, autoreload=autoreload) + _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) + + +if __name__ == "__main__": + _serve() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/state.json b/docs/source-app/workflows/add_web_ui/panel/examples/state.json new file mode 100644 index 0000000000000..cb7d2190bb95e --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/state.json @@ -0,0 +1 @@ +{"vars": {"_paths": {}, "_layout": {"target": "http://localhost:56891/root.flow"}, "requests": [{"id": "my_first_run", "train": {"github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", "script_path": "train_script.py", "script_args": ["--trainer.max_epochs=5", "--trainer.limit_train_batches=4", "--trainer.limit_val_batches=4", "--trainer.callbacks=ModelCheckpoint", "--trainer.callbacks.monitor=val_acc"], "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], "ml_framework": "PyTorch Lightning"}, "best_model_score": 0.21875, "best_model_path": "root.flow.ws.w_0.pt"}]}, "calls": {}, "flows": {}, "works": {}, "structures": {"ws": {"works": {"w_0": {"vars": {"_url": "http://127.0.0.1:56955", "logs": "\rSanity Checking: 0it [00:00, ?it/s]\rSanity Checking: 0%| | 0/2 [00:00 None: port, host, ) + # Todo: Don't log to file when developing locally. Makes it harder to debug. std_err_out = get_frontend_logfile("error.log") std_out_out = get_frontend_logfile("output.log") with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py index 23de1d47d8431..f623223cd651f 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py @@ -70,9 +70,13 @@ def _serve(): url = os.environ["LIGHTNING_FLOW_NAME"] websocket_origin = _get_websocket_origin() + # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 + # Todo: With lightning, the server autoreloads but the browser does not. Fix this. + autoreload = os.environ.get("PANEL_AUTORELOAD", "no")=="yes" + view = _get_view() - pn.serve({url: view}, address=address, port=port, websocket_origin=websocket_origin, show=False) + pn.serve({url: view}, address=address, port=port, websocket_origin=websocket_origin, show=False, autoreload=autoreload) _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 5643f3a26c6b8..6923a731a96e9 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -44,9 +44,14 @@ async def update_fn(): while True: await websocket.recv() # Note: I have not seen use cases where the two lines below are needed - # Note: Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # Also the implementation can make the app state get behind because only 1 update + # is received per 0.2 second (or 1 second). # while (time.time() - last_updated) < 0.2: # time.sleep(0.05) + + # Todo: Add some kind of throttling. If 10 messages are received within 100ms then + # there is no need to trigger the app state changed, request state and update 10 times. _logger.debug("App State Changed. Running callbacks") _run_callbacks() diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 4515e05ddc03c..dabf0406f3623 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -79,7 +79,7 @@ def __init__(self): self.param.state.allow_None = False self._initilized = True - # The below was observed when using mocking during testing + # The below was observed when using mocks during testing if not self.state: raise Exception(".state has not been set.") if not self.state._state: @@ -94,6 +94,8 @@ def _get_flow_state(self) -> AppState: return get_flow_state(flow) def _update_flow_state(self): + # Todo: Consider whether to only update if ._state changed + # this might be much more performent with param.edit_constant(self): self.state = self._get_flow_state() _logger.debug("Request app state") From 2d11c23e39fd50ae7b99ad956be643331ddc997c Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Mon, 11 Jul 2022 06:26:59 +0200 Subject: [PATCH 015/103] add more support for autoreload --- .../examples/panel_serve_render_fn_or_file.py | 7 +++- .../panel/panel_serve_render_fn_or_file.py | 19 ++++++++-- .../frontend/utilities/app_state_comm.py | 5 +-- .../frontend/panel/test_panel_serve.py | 35 +++++++++++++++++++ .../panel/test_panel_serve_render_file.py | 6 +++- .../panel/test_panel_serve_render_fn.py | 8 +++-- 6 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 tests/tests_app/frontend/panel/test_panel_serve.py diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py index d60b9ab9437b0..3b7e697709248 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py @@ -63,13 +63,18 @@ def _get_view(): return os.environ["LIGHTNING_RENDER_FILE"] return _get_view_fn() +def _has_autoreload()->bool: + return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "true"] def _serve(): port = int(os.environ["LIGHTNING_RENDER_PORT"]) address = os.environ["LIGHTNING_RENDER_ADDRESS"] url = os.environ["LIGHTNING_FLOW_NAME"] websocket_origin = _get_websocket_origin() - autoreload = os.environ.get("PANEL_AUTORELOAD", "no")=="yes" + + # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 + # Todo: With lightning, the server autoreloads but the browser does not. Fix this. + autoreload = _has_autoreload() view = _get_view() diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py index f623223cd651f..5874add1b3b3f 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py @@ -36,22 +36,26 @@ _logger = logging.getLogger(__name__) + def _get_render_fn(): render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] return get_render_fn_from_environment(render_fn_name, render_fn_module_file) + def _render_fn_wrapper(): render_fn = _get_render_fn() app = AppStateWatcher() return render_fn(app) + def _get_view_fn(): render_fn = _get_render_fn() if inspect.signature(render_fn).parameters: return _render_fn_wrapper return render_fn + def _get_websocket_origin() -> str: # Todo: Improve this. I don't know how to find the specific host(s). # I tried but it did not work in cloud @@ -64,6 +68,10 @@ def _get_view(): return _get_view_fn() +def _has_autoreload() -> bool: + return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] + + def _serve(): port = int(os.environ["LIGHTNING_RENDER_PORT"]) address = os.environ["LIGHTNING_RENDER_ADDRESS"] @@ -72,11 +80,18 @@ def _serve(): # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 # Todo: With lightning, the server autoreloads but the browser does not. Fix this. - autoreload = os.environ.get("PANEL_AUTORELOAD", "no")=="yes" + autoreload = _has_autoreload() view = _get_view() - pn.serve({url: view}, address=address, port=port, websocket_origin=websocket_origin, show=False, autoreload=autoreload) + pn.serve( + {url: view}, + address=address, + port=port, + websocket_origin=websocket_origin, + show=False, + autoreload=autoreload, + ) _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 6923a731a96e9..82e57673e48ad 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -49,9 +49,10 @@ async def update_fn(): # is received per 0.2 second (or 1 second). # while (time.time() - last_updated) < 0.2: # time.sleep(0.05) - + # Todo: Add some kind of throttling. If 10 messages are received within 100ms then - # there is no need to trigger the app state changed, request state and update 10 times. + # there is no need to trigger the app state changed, request state and update + # 10 times. _logger.debug("App State Changed. Running callbacks") _run_callbacks() diff --git a/tests/tests_app/frontend/panel/test_panel_serve.py b/tests/tests_app/frontend/panel/test_panel_serve.py new file mode 100644 index 0000000000000..f78d44fe86915 --- /dev/null +++ b/tests/tests_app/frontend/panel/test_panel_serve.py @@ -0,0 +1,35 @@ +"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with +Lightning.""" +import os +from unittest import mock + +import pytest + +from lightning_app.frontend.panel.panel_serve_render_fn_or_file import _has_autoreload + + +@pytest.mark.parametrize( + ["value", "expected"], + ( + ("Yes", True), + ("yes", True), + ("YES", True), + ("Y", True), + ("y", True), + ("True", True), + ("true", True), + ("TRUE", True), + ("No", False), + ("no", False), + ("NO", False), + ("N", False), + ("n", False), + ("False", False), + ("false", False), + ("FALSE", False), + ), +) +def test_autoreload(value, expected): + """We can get and set autoreload via the environment variable PANEL_AUTORELOAD""" + with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): + assert _has_autoreload() == expected diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_file.py b/tests/tests_app/frontend/panel/test_panel_serve_render_file.py index 0b3e99394e7c6..2487c0eee7993 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_file.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_file.py @@ -1,5 +1,8 @@ """The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with -Lightning.""" +Lightning. + +These tests are for serving a render_file script or notebook. +""" # pylint: disable=redefined-outer-name import os import pathlib @@ -44,4 +47,5 @@ def test_serve(pn_serve: mock.MagicMock, render_file): port=61896, websocket_origin="*", show=False, + autoreload=False, ) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 0085f8cb47053..9beeb301e59a1 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -1,14 +1,17 @@ """The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with -Lightning.""" +Lightning. + +These tests are for serving a render_fn function. +""" import os from unittest import mock import pytest from lightning_app.frontend.panel.panel_serve_render_fn_or_file import ( - _serve, _get_view_fn, _render_fn_wrapper, + _serve, ) from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher @@ -76,4 +79,5 @@ def test_serve(pn_serve: mock.MagicMock): port=61896, websocket_origin="*", show=False, + autoreload=False, ) From 7f94c4a3461b941f31166e4a90e21b94d8bd76a4 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Mon, 11 Jul 2022 11:10:27 +0200 Subject: [PATCH 016/103] Make autoreload work great for panel .py and .ipynb files --- .../panel/examples/app_state_comm.py | 8 ++- .../panel/examples/app_state_watcher.py | 4 +- .../add_web_ui/panel/examples/other.py | 15 ++++++ .../panel/examples/panel_frontend.py | 53 ++++++++++++++++--- .../examples/panel_serve_render_fn_or_file.py | 33 ++++++++---- .../frontend/panel/panel_frontend.py | 46 +++++++++++++--- .../panel/panel_serve_render_fn_or_file.py | 18 +++---- src/lightning_app/frontend/utilities/other.py | 15 ++++++ .../frontend/panel/test_panel_serve.py | 6 +-- .../panel/test_panel_serve_render_file.py | 51 ------------------ .../frontend/utilities/test_other.py | 32 +++++++++++ 11 files changed, 191 insertions(+), 90 deletions(-) delete mode 100644 tests/tests_app/frontend/panel/test_panel_serve_render_file.py diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py index 5643f3a26c6b8..82e57673e48ad 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py @@ -44,9 +44,15 @@ async def update_fn(): while True: await websocket.recv() # Note: I have not seen use cases where the two lines below are needed - # Note: Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # Also the implementation can make the app state get behind because only 1 update + # is received per 0.2 second (or 1 second). # while (time.time() - last_updated) < 0.2: # time.sleep(0.05) + + # Todo: Add some kind of throttling. If 10 messages are received within 100ms then + # there is no need to trigger the app state changed, request state and update + # 10 times. _logger.debug("App State Changed. Running callbacks") _run_callbacks() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py index c587fd75b32db..7fc9f493065c9 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py @@ -79,7 +79,7 @@ def __init__(self): self.param.state.allow_None = False self._initilized = True - # The below was observed when using mocking during testing + # The below was observed when using mocks during testing if not self.state: raise Exception(".state has not been set.") if not self.state._state: @@ -94,6 +94,8 @@ def _get_flow_state(self) -> AppState: return get_flow_state(flow) def _update_flow_state(self): + # Todo: Consider whether to only update if ._state changed + # this might be much more performent with param.edit_constant(self): self.state = self._get_flow_state() _logger.debug("Request app state") diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/other.py b/docs/source-app/workflows/add_web_ui/panel/examples/other.py index a8b0e1b6c82cd..96746bb945a5c 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/other.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/other.py @@ -40,6 +40,21 @@ def get_flow_state(flow: str) -> AppState: return flow_state +def get_allowed_hosts() -> str: + """Returns a comma separated list of host[:port] that should be allowed to connect""" + # Todo: Improve this. I don't know how to find the specific host(s). + # I tried but it did not work in cloud + return "*" + + +def has_panel_autoreload() -> bool: + """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. + + Please note the casing of value does not matter + """ + return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] + + def get_frontend_environment( flow: str, render_fn_or_file: Callable | str, port: int, host: str ) -> os._Environ: diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py index 659940310e00c..c978f53689ac0 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -6,10 +6,14 @@ import pathlib import subprocess import sys -from typing import Callable +from typing import Callable, List from lightning_app.frontend.frontend import Frontend -from other import get_frontend_environment +from other import ( + get_allowed_hosts, + get_frontend_environment, + has_panel_autoreload, +) from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -17,6 +21,7 @@ class PanelFrontend(Frontend): + # Todo: Add Example to docstring """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. To use this frontend, you must first install the `panel` package: @@ -28,7 +33,8 @@ class PanelFrontend(Frontend): Please note the Panel server will be logging output to error.log and output.log files respectively. - # Todo: Add Example + You can start the lightning server with Panel autoreload by setting the `PANEL_AUTORELOAD` + environment variable to 'yes': `AUTORELOAD=yes lightning run app my_app.py`. Args: render_fn_or_file: A pure function or the path to a .py or .ipynb file. @@ -42,6 +48,8 @@ class PanelFrontend(Frontend): @requires("panel") def __init__(self, render_fn_or_file: Callable | str): + # Todo: consider renaming back to render_fn or something else short. + # Its a hazzle reading and writing such a long name super().__init__() if inspect.ismethod(render_fn_or_file): @@ -54,6 +62,38 @@ def __init__(self, render_fn_or_file: Callable | str): self._process: None | subprocess.Popen = None _logger.debug("initialized") + def _get_popen_args(self, host: str, port: int) -> List: + if isinstance(self.render_fn_or_file, str): + path=pathlib.Path(self.render_fn_or_file) + abs_path = str(path) + # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} + # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and + # seems to work fine. + command = [ + sys.executable, + "-m", + "panel", + "serve", + abs_path, + "--port", + str(port), + "--address", + host, + "--prefix", + self.flow.name, + "--allow-websocket-origin", + get_allowed_hosts(), + ] + if has_panel_autoreload(): + command.append("--autoreload") + _logger.debug("%s", command) + return command + + return [ + sys.executable, + pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", + ] + def start_server(self, host: str, port: int) -> None: _logger.debug("starting server %s %s", host, port) env = get_frontend_environment( @@ -62,14 +102,13 @@ def start_server(self, host: str, port: int) -> None: port, host, ) + command = self._get_popen_args(host, port) + # Todo: Don't log to file when developing locally. Makes it harder to debug. std_err_out = get_frontend_logfile("error.log") std_out_out = get_frontend_logfile("output.log") with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: self._process = subprocess.Popen( # pylint: disable=consider-using-with - [ - sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", - ], + command, env=env, # stdout=stdout, # stderr=stderr, diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py index 3b7e697709248..7bd33283926cc 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py @@ -32,53 +32,64 @@ import panel as pn from app_state_watcher import AppStateWatcher -from other import get_render_fn_from_environment +from other import get_allowed_hosts, get_render_fn_from_environment _logger = logging.getLogger(__name__) + def _get_render_fn(): render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] return get_render_fn_from_environment(render_fn_name, render_fn_module_file) + def _render_fn_wrapper(): render_fn = _get_render_fn() app = AppStateWatcher() return render_fn(app) + def _get_view_fn(): render_fn = _get_render_fn() if inspect.signature(render_fn).parameters: return _render_fn_wrapper return render_fn -def _get_websocket_origin() -> str: - # Todo: Improve this. I don't know how to find the specific host(s). - # I tried but it did not work in cloud - return "*" - def _get_view(): if "LIGHTNING_RENDER_FILE" in os.environ: return os.environ["LIGHTNING_RENDER_FILE"] return _get_view_fn() -def _has_autoreload()->bool: - return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "true"] + +def has_panel_autoreload() -> bool: + """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. + + Please note the casing does not matter + """ + return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] + def _serve(): port = int(os.environ["LIGHTNING_RENDER_PORT"]) address = os.environ["LIGHTNING_RENDER_ADDRESS"] url = os.environ["LIGHTNING_FLOW_NAME"] - websocket_origin = _get_websocket_origin() + websocket_origin = get_allowed_hosts() # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 # Todo: With lightning, the server autoreloads but the browser does not. Fix this. - autoreload = _has_autoreload() + autoreload = has_panel_autoreload() view = _get_view() - pn.serve({url: view}, address=address, port=port, websocket_origin=websocket_origin, show=False, autoreload=autoreload) + pn.serve( + {url: view}, + address=address, + port=port, + websocket_origin=websocket_origin, + show=False, + autoreload=autoreload, + ) _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index d9d79cc164023..4e98eb73c38e1 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -6,10 +6,14 @@ import pathlib import subprocess import sys -from typing import Callable +from typing import Callable, List from lightning_app.frontend.frontend import Frontend -from lightning_app.frontend.utilities.other import get_frontend_environment +from lightning_app.frontend.utilities.other import ( + get_allowed_hosts, + get_frontend_environment, + has_panel_autoreload, +) from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -58,6 +62,38 @@ def __init__(self, render_fn_or_file: Callable | str): self._process: None | subprocess.Popen = None _logger.debug("initialized") + def _get_popen_args(self, host: str, port: int) -> List: + if isinstance(self.render_fn_or_file, str): + path = pathlib.Path(self.render_fn_or_file) + abs_path = str(path) + # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} + # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and + # seems to work fine. + command = [ + sys.executable, + "-m", + "panel", + "serve", + abs_path, + "--port", + str(port), + "--address", + host, + "--prefix", + self.flow.name, + "--allow-websocket-origin", + get_allowed_hosts(), + ] + if has_panel_autoreload(): + command.append("--autoreload") + _logger.debug("%s", command) + return command + + return [ + sys.executable, + pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", + ] + def start_server(self, host: str, port: int) -> None: _logger.debug("starting server %s %s", host, port) env = get_frontend_environment( @@ -66,15 +102,13 @@ def start_server(self, host: str, port: int) -> None: port, host, ) + command = self._get_popen_args(host, port) # Todo: Don't log to file when developing locally. Makes it harder to debug. std_err_out = get_frontend_logfile("error.log") std_out_out = get_frontend_logfile("output.log") with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: self._process = subprocess.Popen( # pylint: disable=consider-using-with - [ - sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", - ], + command, env=env, stdout=stdout, stderr=stderr, diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py index 5874add1b3b3f..c02b23d3b9f2a 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py @@ -32,7 +32,7 @@ import panel as pn from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher -from lightning_app.frontend.utilities.other import get_render_fn_from_environment +from lightning_app.frontend.utilities.other import get_allowed_hosts, get_render_fn_from_environment _logger = logging.getLogger(__name__) @@ -56,19 +56,17 @@ def _get_view_fn(): return render_fn -def _get_websocket_origin() -> str: - # Todo: Improve this. I don't know how to find the specific host(s). - # I tried but it did not work in cloud - return "*" - - def _get_view(): if "LIGHTNING_RENDER_FILE" in os.environ: return os.environ["LIGHTNING_RENDER_FILE"] return _get_view_fn() -def _has_autoreload() -> bool: +def has_panel_autoreload() -> bool: + """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. + + Please note the casing does not matter + """ return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] @@ -76,11 +74,11 @@ def _serve(): port = int(os.environ["LIGHTNING_RENDER_PORT"]) address = os.environ["LIGHTNING_RENDER_ADDRESS"] url = os.environ["LIGHTNING_FLOW_NAME"] - websocket_origin = _get_websocket_origin() + websocket_origin = get_allowed_hosts() # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 # Todo: With lightning, the server autoreloads but the browser does not. Fix this. - autoreload = _has_autoreload() + autoreload = has_panel_autoreload() view = _get_view() diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py index a8b0e1b6c82cd..96746bb945a5c 100644 --- a/src/lightning_app/frontend/utilities/other.py +++ b/src/lightning_app/frontend/utilities/other.py @@ -40,6 +40,21 @@ def get_flow_state(flow: str) -> AppState: return flow_state +def get_allowed_hosts() -> str: + """Returns a comma separated list of host[:port] that should be allowed to connect""" + # Todo: Improve this. I don't know how to find the specific host(s). + # I tried but it did not work in cloud + return "*" + + +def has_panel_autoreload() -> bool: + """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. + + Please note the casing of value does not matter + """ + return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] + + def get_frontend_environment( flow: str, render_fn_or_file: Callable | str, port: int, host: str ) -> os._Environ: diff --git a/tests/tests_app/frontend/panel/test_panel_serve.py b/tests/tests_app/frontend/panel/test_panel_serve.py index f78d44fe86915..a1e0258ea2e3e 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve.py +++ b/tests/tests_app/frontend/panel/test_panel_serve.py @@ -5,7 +5,7 @@ import pytest -from lightning_app.frontend.panel.panel_serve_render_fn_or_file import _has_autoreload +from lightning_app.frontend.panel.panel_serve_render_fn_or_file import has_panel_autoreload @pytest.mark.parametrize( @@ -29,7 +29,7 @@ ("FALSE", False), ), ) -def test_autoreload(value, expected): +def test_has_panel_autoreload(value, expected): """We can get and set autoreload via the environment variable PANEL_AUTORELOAD""" with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): - assert _has_autoreload() == expected + assert has_panel_autoreload() == expected diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_file.py b/tests/tests_app/frontend/panel/test_panel_serve_render_file.py deleted file mode 100644 index 2487c0eee7993..0000000000000 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_file.py +++ /dev/null @@ -1,51 +0,0 @@ -"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with -Lightning. - -These tests are for serving a render_file script or notebook. -""" -# pylint: disable=redefined-outer-name -import os -import pathlib -from unittest import mock - -import pytest - -from lightning_app.frontend.panel.panel_serve_render_fn_or_file import _serve - - -@pytest.fixture(scope="module") -def render_file(): - """Returns the path to a Panel app file""" - path = pathlib.Path(__file__).parent / "app_panel.py" - path = path.relative_to(pathlib.Path.cwd()) - return str(path) - - -@pytest.fixture(autouse=True, scope="module") -def mock_settings_env_vars(render_file): - """Set the LIGHTNING environment variables.""" - - with mock.patch.dict( - os.environ, - { - "LIGHTNING_FLOW_NAME": "root.lit_flow", - "LIGHTNING_RENDER_ADDRESS": "localhost", - "LIGHTNING_RENDER_FILE": render_file, - "LIGHTNING_RENDER_PORT": "61896", - }, - ): - yield - - -@mock.patch("panel.serve") -def test_serve(pn_serve: mock.MagicMock, render_file): - """We can run python panel_serve_render_fn_or_file to serve the render_file.""" - _serve() - pn_serve.assert_called_once_with( - {"root.lit_flow": render_file}, - address="localhost", - port=61896, - websocket_origin="*", - show=False, - autoreload=False, - ) diff --git a/tests/tests_app/frontend/utilities/test_other.py b/tests/tests_app/frontend/utilities/test_other.py index da480ae0f8a41..59c6159c82306 100644 --- a/tests/tests_app/frontend/utilities/test_other.py +++ b/tests/tests_app/frontend/utilities/test_other.py @@ -1,10 +1,15 @@ """We have some utility functions that can be used across frontends.""" import inspect +import os +from unittest import mock + +import pytest from lightning_app.frontend.utilities.other import ( get_flow_state, get_frontend_environment, get_render_fn_from_environment, + has_panel_autoreload, ) from lightning_app.utilities.state import AppState @@ -60,3 +65,30 @@ def test_get_frontend_environment_file(): assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" assert env["LIGHTNING_RENDER_FILE"] == "app_panel.py" assert env["LIGHTNING_RENDER_PORT"] == "1234" + + +@pytest.mark.parametrize( + ["value", "expected"], + ( + ("Yes", True), + ("yes", True), + ("YES", True), + ("Y", True), + ("y", True), + ("True", True), + ("true", True), + ("TRUE", True), + ("No", False), + ("no", False), + ("NO", False), + ("N", False), + ("n", False), + ("False", False), + ("false", False), + ("FALSE", False), + ), +) +def test_has_panel_autoreload(value, expected): + """We can get and set autoreload via the environment variable PANEL_AUTORELOAD""" + with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): + assert has_panel_autoreload() == expected From 7f38248e84fc81ea4fc645326dcfe5bf1eb9ca74 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Mon, 11 Jul 2022 20:36:37 +0200 Subject: [PATCH 017/103] panel serve render_fn too --- .../add_web_ui/panel/examples/app_basic.py | 3 +- .../panel/examples/panel_frontend.py | 58 ++++++----- .../panel/examples/panel_serve_render_fn.py | 42 ++++++++ .../examples/panel_serve_render_fn_or_file.py | 97 ------------------- .../frontend/panel/panel_frontend.py | 52 +++++----- .../frontend/panel/panel_serve_render_fn.py | 42 ++++++++ .../panel/panel_serve_render_fn_or_file.py | 97 ------------------- .../frontend/panel/test_panel_serve.py | 35 ------- .../panel/test_panel_serve_render_fn.py | 49 +++------- 9 files changed, 154 insertions(+), 321 deletions(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py create mode 100644 src/lightning_app/frontend/panel/panel_serve_render_fn.py delete mode 100644 src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py delete mode 100644 tests/tests_app/frontend/panel/test_panel_serve.py diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py index 872dfada30e23..e23efeacdcc52 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py @@ -8,8 +8,7 @@ def your_panel_app(app): - return pn.pane.Markdown("hello") - + pn.pane.Markdown("hello").servable() class LitPanel(L.LightningFlow): def __init__(self): diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py index c978f53689ac0..6d6a764003c5d 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -63,36 +63,34 @@ def __init__(self, render_fn_or_file: Callable | str): _logger.debug("initialized") def _get_popen_args(self, host: str, port: int) -> List: - if isinstance(self.render_fn_or_file, str): - path=pathlib.Path(self.render_fn_or_file) - abs_path = str(path) - # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} - # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and - # seems to work fine. - command = [ - sys.executable, - "-m", - "panel", - "serve", - abs_path, - "--port", - str(port), - "--address", - host, - "--prefix", - self.flow.name, - "--allow-websocket-origin", - get_allowed_hosts(), - ] - if has_panel_autoreload(): - command.append("--autoreload") - _logger.debug("%s", command) - return command - - return [ + if callable(self.render_fn_or_file): + path = str(pathlib.Path(__file__).parent / "panel_serve_render_fn.py") + else: + path = pathlib.Path(self.render_fn_or_file) + + abs_path = str(path) + # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} + # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and + # seems to work fine. + command = [ sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", + "-m", + "panel", + "serve", + abs_path, + "--port", + str(port), + "--address", + host, + "--prefix", + self.flow.name, + "--allow-websocket-origin", + get_allowed_hosts(), ] + if has_panel_autoreload(): + command.append("--autoreload") + _logger.debug("%s", command) + return command def start_server(self, host: str, port: int) -> None: _logger.debug("starting server %s %s", host, port) @@ -110,8 +108,8 @@ def start_server(self, host: str, port: int) -> None: self._process = subprocess.Popen( # pylint: disable=consider-using-with command, env=env, - # stdout=stdout, - # stderr=stderr, + stdout=stdout, + stderr=stderr, ) def stop_server(self) -> None: diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py new file mode 100644 index 0000000000000..dcc83d22ded0e --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py @@ -0,0 +1,42 @@ +"""This file gets run by Python to lunch a Panel Server with Lightning. + +From here, we will call the render_fn that the user provided to the PanelFrontend. + +It requires the below environment variables to be set + +- LIGHTNING_RENDER_FUNCTION +- LIGHTNING_RENDER_MODULE_FILE + +Example: + +.. code-block:: bash + + python panel_serve_render_fn +""" +import inspect +import os + +import panel as pn +from app_state_watcher import AppStateWatcher +from other import get_render_fn_from_environment + +def _get_render_fn(): + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + render_fn = get_render_fn_from_environment(render_fn_name, render_fn_module_file) + if inspect.signature(render_fn).parameters: + + def _render_fn_wrapper(): + app = AppStateWatcher() + return render_fn(app) + + return _render_fn_wrapper + return render_fn + + +if __name__.startswith("bokeh"): + # I use caching for efficiency reasons. It shaves off 10ms from having + # to get_render_fn_from_environment every time + if not "lightning_render_fn" in pn.state.cache: + pn.state.cache["lightning_render_fn"] = _get_render_fn() + pn.state.cache["lightning_render_fn"]() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py deleted file mode 100644 index 7bd33283926cc..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn_or_file.py +++ /dev/null @@ -1,97 +0,0 @@ -"""This file gets run by Python to lunch a Panel Server with Lightning. - -From here, we will call the render_fn that the user provided to the PanelFrontend. - -It requires the below environment variables to be set - -- LIGHTNING_FLOW_NAME -- LIGHTNING_RENDER_ADDRESS -- LIGHTNING_RENDER_PORT - -As well as either - -- LIGHTNING_RENDER_FUNCTION + LIGHTNING_RENDER_MODULE_FILE - -or - -- LIGHTNING_RENDER_FILE - - -Example: - -.. code-block:: bash - - python panel_serve_render_fn_or_file -""" -from __future__ import annotations - -import inspect -import logging -import os - -import panel as pn - -from app_state_watcher import AppStateWatcher -from other import get_allowed_hosts, get_render_fn_from_environment - -_logger = logging.getLogger(__name__) - - -def _get_render_fn(): - render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] - render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] - return get_render_fn_from_environment(render_fn_name, render_fn_module_file) - - -def _render_fn_wrapper(): - render_fn = _get_render_fn() - app = AppStateWatcher() - return render_fn(app) - - -def _get_view_fn(): - render_fn = _get_render_fn() - if inspect.signature(render_fn).parameters: - return _render_fn_wrapper - return render_fn - - -def _get_view(): - if "LIGHTNING_RENDER_FILE" in os.environ: - return os.environ["LIGHTNING_RENDER_FILE"] - return _get_view_fn() - - -def has_panel_autoreload() -> bool: - """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. - - Please note the casing does not matter - """ - return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] - - -def _serve(): - port = int(os.environ["LIGHTNING_RENDER_PORT"]) - address = os.environ["LIGHTNING_RENDER_ADDRESS"] - url = os.environ["LIGHTNING_FLOW_NAME"] - websocket_origin = get_allowed_hosts() - - # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 - # Todo: With lightning, the server autoreloads but the browser does not. Fix this. - autoreload = has_panel_autoreload() - - view = _get_view() - - pn.serve( - {url: view}, - address=address, - port=port, - websocket_origin=websocket_origin, - show=False, - autoreload=autoreload, - ) - _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) - - -if __name__ == "__main__": - _serve() diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 4e98eb73c38e1..5dd7e50c1653f 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -63,36 +63,34 @@ def __init__(self, render_fn_or_file: Callable | str): _logger.debug("initialized") def _get_popen_args(self, host: str, port: int) -> List: - if isinstance(self.render_fn_or_file, str): + if callable(self.render_fn_or_file): + path = str(pathlib.Path(__file__).parent / "panel_serve_render_fn.py") + else: path = pathlib.Path(self.render_fn_or_file) - abs_path = str(path) - # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} - # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and - # seems to work fine. - command = [ - sys.executable, - "-m", - "panel", - "serve", - abs_path, - "--port", - str(port), - "--address", - host, - "--prefix", - self.flow.name, - "--allow-websocket-origin", - get_allowed_hosts(), - ] - if has_panel_autoreload(): - command.append("--autoreload") - _logger.debug("%s", command) - return command - - return [ + + abs_path = str(path) + # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} + # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and + # seems to work fine. + command = [ sys.executable, - pathlib.Path(__file__).parent / "panel_serve_render_fn_or_file.py", + "-m", + "panel", + "serve", + abs_path, + "--port", + str(port), + "--address", + host, + "--prefix", + self.flow.name, + "--allow-websocket-origin", + get_allowed_hosts(), ] + if has_panel_autoreload(): + command.append("--autoreload") + _logger.debug("%s", command) + return command def start_server(self, host: str, port: int) -> None: _logger.debug("starting server %s %s", host, port) diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py new file mode 100644 index 0000000000000..ff9140ba8b9f7 --- /dev/null +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -0,0 +1,42 @@ +"""This file gets run by Python to lunch a Panel Server with Lightning. + +From here, we will call the render_fn that the user provided to the PanelFrontend. + +It requires the below environment variables to be set + +- LIGHTNING_RENDER_FUNCTION +- LIGHTNING_RENDER_MODULE_FILE + +Example: + +.. code-block:: bash + + python panel_serve_render_fn +""" +import inspect +import os + +import panel as pn + +from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher +from lightning_app.frontend.utilities.other import get_render_fn_from_environment + +def _get_render_fn(): + render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] + render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] + render_fn = get_render_fn_from_environment(render_fn_name, render_fn_module_file) + if inspect.signature(render_fn).parameters: + + def _render_fn_wrapper(): + app = AppStateWatcher() + return render_fn(app) + + return _render_fn_wrapper + return render_fn + +if __name__.startswith("bokeh"): + # I use caching for efficiency reasons. It shaves off 10ms from having + # to get_render_fn_from_environment every time + if not "lightning_render_fn" in pn.state.cache: + pn.state.cache["lightning_render_fn"] = _get_render_fn() + pn.state.cache["lightning_render_fn"]() diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py b/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py deleted file mode 100644 index c02b23d3b9f2a..0000000000000 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn_or_file.py +++ /dev/null @@ -1,97 +0,0 @@ -"""This file gets run by Python to lunch a Panel Server with Lightning. - -From here, we will call the render_fn that the user provided to the PanelFrontend. - -It requires the below environment variables to be set - -- LIGHTNING_FLOW_NAME -- LIGHTNING_RENDER_ADDRESS -- LIGHTNING_RENDER_PORT - -As well as either - -- LIGHTNING_RENDER_FUNCTION + LIGHTNING_RENDER_MODULE_FILE - -or - -- LIGHTNING_RENDER_FILE - - -Example: - -.. code-block:: bash - - python panel_serve_render_fn_or_file -""" -from __future__ import annotations - -import inspect -import logging -import os - -import panel as pn - -from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher -from lightning_app.frontend.utilities.other import get_allowed_hosts, get_render_fn_from_environment - -_logger = logging.getLogger(__name__) - - -def _get_render_fn(): - render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] - render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] - return get_render_fn_from_environment(render_fn_name, render_fn_module_file) - - -def _render_fn_wrapper(): - render_fn = _get_render_fn() - app = AppStateWatcher() - return render_fn(app) - - -def _get_view_fn(): - render_fn = _get_render_fn() - if inspect.signature(render_fn).parameters: - return _render_fn_wrapper - return render_fn - - -def _get_view(): - if "LIGHTNING_RENDER_FILE" in os.environ: - return os.environ["LIGHTNING_RENDER_FILE"] - return _get_view_fn() - - -def has_panel_autoreload() -> bool: - """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. - - Please note the casing does not matter - """ - return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] - - -def _serve(): - port = int(os.environ["LIGHTNING_RENDER_PORT"]) - address = os.environ["LIGHTNING_RENDER_ADDRESS"] - url = os.environ["LIGHTNING_FLOW_NAME"] - websocket_origin = get_allowed_hosts() - - # PANEL_AUTORELOAD not yet supported by Panel. See https://github.com/holoviz/panel/issues/3681 - # Todo: With lightning, the server autoreloads but the browser does not. Fix this. - autoreload = has_panel_autoreload() - - view = _get_view() - - pn.serve( - {url: view}, - address=address, - port=port, - websocket_origin=websocket_origin, - show=False, - autoreload=autoreload, - ) - _logger.debug("Panel server started on port http://%s:%s/%s", address, port, url) - - -if __name__ == "__main__": - _serve() diff --git a/tests/tests_app/frontend/panel/test_panel_serve.py b/tests/tests_app/frontend/panel/test_panel_serve.py deleted file mode 100644 index a1e0258ea2e3e..0000000000000 --- a/tests/tests_app/frontend/panel/test_panel_serve.py +++ /dev/null @@ -1,35 +0,0 @@ -"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with -Lightning.""" -import os -from unittest import mock - -import pytest - -from lightning_app.frontend.panel.panel_serve_render_fn_or_file import has_panel_autoreload - - -@pytest.mark.parametrize( - ["value", "expected"], - ( - ("Yes", True), - ("yes", True), - ("YES", True), - ("Y", True), - ("y", True), - ("True", True), - ("true", True), - ("TRUE", True), - ("No", False), - ("no", False), - ("NO", False), - ("N", False), - ("n", False), - ("False", False), - ("false", False), - ("FALSE", False), - ), -) -def test_has_panel_autoreload(value, expected): - """We can get and set autoreload via the environment variable PANEL_AUTORELOAD""" - with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): - assert has_panel_autoreload() == expected diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 9beeb301e59a1..3852613f8cd96 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -8,23 +8,17 @@ import pytest -from lightning_app.frontend.panel.panel_serve_render_fn_or_file import ( - _get_view_fn, - _render_fn_wrapper, - _serve, -) +from lightning_app.frontend.panel.panel_serve_render_fn import _get_render_fn from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher -@pytest.fixture(autouse=True, scope="module") -def mock_settings_env_vars(): - """Set the LIGHTNING environment variables.""" +@pytest.fixture(autouse=True) +def _mock_settings_env_vars(): with mock.patch.dict( os.environ, { "LIGHTNING_FLOW_NAME": "root.lit_flow", "LIGHTNING_RENDER_ADDRESS": "localhost", - "LIGHTNING_RENDER_FUNCTION": "render_fn", "LIGHTNING_RENDER_MODULE_FILE": __file__, "LIGHTNING_RENDER_PORT": "61896", }, @@ -33,30 +27,34 @@ def mock_settings_env_vars(): def render_fn(app): - """Test function that just passes through the app.""" + """Test render_fn function with app args.""" return app -def test_get_view_fn(): +@mock.patch.dict( + os.environ, + { + "LIGHTNING_RENDER_FUNCTION": "render_fn", + }, +) +def test_get_view_fn_args(): """We have a helper get_view_fn function that create a function for our view. If the render_fn provides an argument an AppStateWatcher is provided as argument """ - view_fn = _get_view_fn() - result = view_fn() - assert isinstance(result, AppStateWatcher) + result = _get_render_fn() + assert isinstance(result(), AppStateWatcher) def render_fn_no_args(): """Test function with no arguments""" - return "Hello" + return "no_args" @mock.patch.dict( os.environ, { "LIGHTNING_RENDER_FUNCTION": "render_fn_no_args", - "LIGHTNING_RENDER_MODULE_FILE": __file__, }, ) def test_get_view_fn_no_args(): @@ -64,20 +62,5 @@ def test_get_view_fn_no_args(): If the render_fn provides an argument an AppStateWatcher is provided as argument """ - view_fn = _get_view_fn() - result = view_fn() - assert result == "Hello" - - -@mock.patch("panel.serve") -def test_serve(pn_serve: mock.MagicMock): - """We can run python panel_serve_render_fn_or_file to serve the render_fn.""" - _serve() - pn_serve.assert_called_once_with( - {"root.lit_flow": _render_fn_wrapper}, - address="localhost", - port=61896, - websocket_origin="*", - show=False, - autoreload=False, - ) + result = _get_render_fn() + assert result() == "no_args" From 1722e4fba6ad9269f0f05a9c591a79f3a48e488a Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Tue, 12 Jul 2022 08:28:00 +0200 Subject: [PATCH 018/103] dont use log files when running locally --- .../add_web_ui/panel/examples/other.py | 9 ++++ .../panel/examples/panel_frontend.py | 45 +++++++++++------- .../panel/examples/panel_serve_render_fn.py | 2 + .../frontend/panel/panel_frontend.py | 39 +++++++++++----- .../frontend/panel/panel_serve_render_fn.py | 2 + src/lightning_app/frontend/utilities/other.py | 9 ++++ .../frontend/panel/test_panel_frontend.py | 46 ++++++++++++++++--- .../frontend/utilities/test_other.py | 13 ++++++ 8 files changed, 130 insertions(+), 35 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/other.py b/docs/source-app/workflows/add_web_ui/panel/examples/other.py index 96746bb945a5c..d5a650e100e6a 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/other.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/other.py @@ -81,3 +81,12 @@ def get_frontend_environment( env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn_or_file).__file__ return env + + +def is_running_locally() -> bool: + """Returns True if the lightning app is running locally. + + This function can be used to determine if the app is running locally and provide a better + developer experience. + """ + return "LIGHTNING_APP_STATE_URL" not in os.environ diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py index 6d6a764003c5d..7120d811be994 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -6,14 +6,11 @@ import pathlib import subprocess import sys -from typing import Callable, List +from typing import Callable, Dict, List + +from other import get_allowed_hosts, get_frontend_environment, has_panel_autoreload, is_running_locally from lightning_app.frontend.frontend import Frontend -from other import ( - get_allowed_hosts, - get_frontend_environment, - has_panel_autoreload, -) from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -40,7 +37,6 @@ class PanelFrontend(Frontend): render_fn_or_file: A pure function or the path to a .py or .ipynb file. The function must be a pure function that contains your Panel code. The function can optionally accept an `AppStateWatcher` argument. - The function must return a Panel Viewable. Raises: TypeError: Raised if the render_fn_or_file is a class method @@ -60,6 +56,7 @@ def __init__(self, render_fn_or_file: Callable | str): self.render_fn_or_file = render_fn_or_file self._process: None | subprocess.Popen = None + self._log_files: Dict[str] = {} _logger.debug("initialized") def _get_popen_args(self, host: str, port: int) -> List: @@ -101,18 +98,32 @@ def start_server(self, host: str, port: int) -> None: host, ) command = self._get_popen_args(host, port) - # Todo: Don't log to file when developing locally. Makes it harder to debug. - std_err_out = get_frontend_logfile("error.log") - std_out_out = get_frontend_logfile("output.log") - with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: - self._process = subprocess.Popen( # pylint: disable=consider-using-with - command, - env=env, - stdout=stdout, - stderr=stderr, - ) + + if not is_running_locally(): + self._open_log_files() + + self._process = subprocess.Popen( # pylint: disable=consider-using-with + command, env=env, **self._log_files + ) def stop_server(self) -> None: if self._process is None: raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") self._process.kill() + self._close_log_files() + + def _close_log_files(self): + for file_ in self._log_files.values(): + if not file_.closed: + file_.close() + self._log_files = {} + + def _open_log_files(self) -> Dict: + # Don't log to file when developing locally. Makes it harder to debug. + self._close_log_files() + + std_err_out = get_frontend_logfile("error.log") + std_out_out = get_frontend_logfile("output.log") + stderr = std_err_out.open("wb") + stdout = std_out_out.open("wb") + self._log_files = {"stdout": stderr, "stderr": stdout} diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py index dcc83d22ded0e..4abe311da62f7 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py @@ -17,9 +17,11 @@ import os import panel as pn + from app_state_watcher import AppStateWatcher from other import get_render_fn_from_environment + def _get_render_fn(): render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 5dd7e50c1653f..e18b7f65ae823 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -6,13 +6,14 @@ import pathlib import subprocess import sys -from typing import Callable, List +from typing import Callable, Dict, List from lightning_app.frontend.frontend import Frontend from lightning_app.frontend.utilities.other import ( get_allowed_hosts, get_frontend_environment, has_panel_autoreload, + is_running_locally, ) from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -40,7 +41,6 @@ class PanelFrontend(Frontend): render_fn_or_file: A pure function or the path to a .py or .ipynb file. The function must be a pure function that contains your Panel code. The function can optionally accept an `AppStateWatcher` argument. - The function must return a Panel Viewable. Raises: TypeError: Raised if the render_fn_or_file is a class method @@ -60,6 +60,7 @@ def __init__(self, render_fn_or_file: Callable | str): self.render_fn_or_file = render_fn_or_file self._process: None | subprocess.Popen = None + self._log_files: Dict[str] = {} _logger.debug("initialized") def _get_popen_args(self, host: str, port: int) -> List: @@ -101,18 +102,32 @@ def start_server(self, host: str, port: int) -> None: host, ) command = self._get_popen_args(host, port) - # Todo: Don't log to file when developing locally. Makes it harder to debug. - std_err_out = get_frontend_logfile("error.log") - std_out_out = get_frontend_logfile("output.log") - with open(std_err_out, "wb") as stderr, open(std_out_out, "wb") as stdout: - self._process = subprocess.Popen( # pylint: disable=consider-using-with - command, - env=env, - stdout=stdout, - stderr=stderr, - ) + + if not is_running_locally(): + self._open_log_files() + + self._process = subprocess.Popen( # pylint: disable=consider-using-with + command, env=env, **self._log_files + ) def stop_server(self) -> None: if self._process is None: raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") self._process.kill() + self._close_log_files() + + def _close_log_files(self): + for file_ in self._log_files.values(): + if not file_.closed: + file_.close() + self._log_files = {} + + def _open_log_files(self) -> Dict: + # Don't log to file when developing locally. Makes it harder to debug. + self._close_log_files() + + std_err_out = get_frontend_logfile("error.log") + std_out_out = get_frontend_logfile("output.log") + stderr = std_err_out.open("wb") + stdout = std_out_out.open("wb") + self._log_files = {"stdout": stderr, "stderr": stdout} diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py index ff9140ba8b9f7..f13354fbfea16 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -21,6 +21,7 @@ from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher from lightning_app.frontend.utilities.other import get_render_fn_from_environment + def _get_render_fn(): render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] @@ -34,6 +35,7 @@ def _render_fn_wrapper(): return _render_fn_wrapper return render_fn + if __name__.startswith("bokeh"): # I use caching for efficiency reasons. It shaves off 10ms from having # to get_render_fn_from_environment every time diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py index 96746bb945a5c..d5a650e100e6a 100644 --- a/src/lightning_app/frontend/utilities/other.py +++ b/src/lightning_app/frontend/utilities/other.py @@ -81,3 +81,12 @@ def get_frontend_environment( env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn_or_file).__file__ return env + + +def is_running_locally() -> bool: + """Returns True if the lightning app is running locally. + + This function can be used to determine if the app is running locally and provide a better + developer experience. + """ + return "LIGHTNING_APP_STATE_URL" not in os.environ diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index efd77f7ed5381..5b38627386c65 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -9,7 +9,7 @@ import pytest from lightning_app import LightningFlow -from lightning_app.frontend.panel import PanelFrontend +from lightning_app.frontend.panel import panel_serve_render_fn, PanelFrontend from lightning_app.utilities.state import AppState @@ -50,10 +50,21 @@ def test_panel_frontend_start_stop_server(subprocess_mock): env_variables = subprocess_mock.method_calls[0].kwargs["env"] call_args = subprocess_mock.method_calls[0].args[0] - assert call_args[0] == sys.executable - assert call_args[1].exists() - assert str(call_args[1]).endswith("panel_serve_render_fn_or_file.py") - assert len(call_args) == 2 + assert call_args == [ + sys.executable, + "-m", + "panel", + "serve", + panel_serve_render_fn.__file__, + "--port", + "1111", + "--address", + "hostname", + "--prefix", + "root.my.flow", + "--allow-websocket-origin", + "*", + ] assert env_variables["LIGHTNING_FLOW_NAME"] == "root.my.flow" assert env_variables["LIGHTNING_RENDER_ADDRESS"] == "hostname" @@ -88,7 +99,7 @@ def _call_me(state): ) def test_panel_wrapper_calls_render_fn_or_file(*_): """Run the panel_serve_render_fn_or_file""" - runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn_or_file") + runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn") # TODO: find a way to assert that _call_me got called @@ -102,3 +113,26 @@ def _render_fn(self): with pytest.raises(TypeError, match="being a method"): PanelFrontend(render_fn_or_file=_DummyClass()._render_fn) + + +def test_open_close_log_files() -> bool: + """We can open and close the log files""" + frontend = PanelFrontend(_noop_render_fn) + assert not frontend._log_files + # When + frontend._open_log_files() + # Then + stdout = frontend._log_files["stdout"] + stderr = frontend._log_files["stderr"] + assert not stdout.closed + assert not stderr.closed + + # When + frontend._close_log_files() + # Then + assert not frontend._log_files + assert stdout.closed + assert stderr.closed + + # We can close even if not open + frontend._close_log_files() diff --git a/tests/tests_app/frontend/utilities/test_other.py b/tests/tests_app/frontend/utilities/test_other.py index 59c6159c82306..eb278d80a6a73 100644 --- a/tests/tests_app/frontend/utilities/test_other.py +++ b/tests/tests_app/frontend/utilities/test_other.py @@ -10,6 +10,7 @@ get_frontend_environment, get_render_fn_from_environment, has_panel_autoreload, + is_running_locally, ) from lightning_app.utilities.state import AppState @@ -92,3 +93,15 @@ def test_has_panel_autoreload(value, expected): """We can get and set autoreload via the environment variable PANEL_AUTORELOAD""" with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): assert has_panel_autoreload() == expected + + +@mock.patch.dict(os.environ, clear=True) +def test_is_running_locally() -> bool: + """We can determine if lightning is running locally""" + assert is_running_locally() + + +@mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "127.0.0.1"}) +def test_is_running_cloud() -> bool: + """We can determine if lightning is running in cloud""" + assert not is_running_locally() From 0158f3652b187ec6d2e32e56359db3a385787055 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Tue, 12 Jul 2022 16:12:42 +0200 Subject: [PATCH 019/103] refactor github model runner --- ..._github_render.py => app_github_runner.py} | 2 +- .../panel/examples/panel_github_render.py | 186 ------------- .../panel/examples/panel_github_runner.py | 249 ++++++++++++++++++ 3 files changed, 250 insertions(+), 187 deletions(-) rename docs/source-app/workflows/add_web_ui/panel/examples/{app_github_render.py => app_github_runner.py} (96%) delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py similarity index 96% rename from docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py rename to docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py index 4ed0729201b65..72c200ad7647c 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_render.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py @@ -183,7 +183,7 @@ def _handle_request(self, request_id: int, request: Dict): def configure_layout(self): # Create a StreamLit UI for the user to run his Github Repo. - return PanelFrontend("panel_github_render.py") + return PanelFrontend("panel_github_runner.py") class RootFlow(LightningFlow): diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py deleted file mode 100644 index af6e731bd6f1e..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_render.py +++ /dev/null @@ -1,186 +0,0 @@ -import os -from unittest.mock import Mock - -import panel as pn -import param -from app_state_watcher import AppStateWatcher - -from lightning_app.utilities.state import AppState - -def to_str(value): - if isinstance(value, list): - return "\n".join([str(item) for item in value]) - return str(value) - -if "LIGHTNING_FLOW_NAME" in os.environ: - app = AppStateWatcher() -else: - class AppMock(param.Parameterized): - state = param.Parameter() - - app = AppMock(state=Mock()) - import json - - with open("state.json", "r") as fp: - app.state._state = json.load(fp) - app.state.requests = [ - { - "id": 0, - "train": { - "github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", - "script_path": "train_script.py", - "script_args": [ - "--trainer.max_epochs=5", - "--trainer.limit_train_batches=4", - "--trainer.limit_val_batches=4", - "--trainer.callbacks=ModelCheckpoint", - "--trainer.callbacks.monitor=val_acc", - ], - "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], - "ml_framework": "PyTorch Lightning", - }, - } - ] - - -ACCENT = "#792EE5" -LIGHTNING_SPINNER_URL = ( - "https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/spinners/material/" - "bar_chart_lightning_purple.svg" -) -LIGHTNING_SPINNER = pn.pane.HTML( - f"" -) -# Todo: Set JSON theme depending on template theme -# pn.pane.JSON.param.theme.default = "dark" -pn.pane.JSON.param.hover_preview.default = True - -pn.config.raw_css.append( - """ - .bk-root { - height: calc( 100vh - 200px ) !important; - } - """ -) -pn.extension("terminal", sizing_mode="stretch_width", template="fast") -pn.state.template.param.update(accent_base_color=ACCENT, header_background=ACCENT) - - -def create_new_page(): - title = "# Create a new run 🎈" - id_input = pn.widgets.TextInput(name="Name your run", value="my_first_run") - github_repo_input = pn.widgets.TextInput( - name="Enter a Github Repo URL", - value="https://github.com/Lightning-AI/lightning-quick-start.git", - ) - script_path_input = pn.widgets.TextInput( - name="Enter your script to run", value="train_script.py" - ) - - default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" - script_args_input = pn.widgets.TextInput( - name="Enter your base script arguments", value=default_script_args - ) - default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" - requirements_input = pn.widgets.TextInput( - name="Enter your requirements", value=default_requirements - ) - ml_framework_input = pn.widgets.RadioBoxGroup( - name="Select your ML Training Frameworks", - options=["PyTorch Lightning", "Keras", "Tensorflow"], - inline=True, - ) - submit_input = pn.widgets.Button(name="⚡ SUBMIT ⚡", button_type="primary") - - @pn.depends(submit_input, watch=True) - def create_new_run(_): - new_request = { - "id": id_input.value, - "train": { - "github_repo": github_repo_input.value, - "script_path": script_path_input.value, - "script_args": script_args_input.value.split(" "), - "requirements": requirements_input.value.split(" "), - "ml_framework": ml_framework_input.value, - }, - } - app.state.requests = app.state.requests + [new_request] - print("submitted", new_request) - - @pn.depends(ml_framework_input.param.value) - def message_or_button(ml_framework): - if ml_framework not in ("PyTorch Lightning"): - return f"{ml_framework} isn't supported yet." - else: - return submit_input - - return pn.Column( - title, - id_input, - github_repo_input, - script_path_input, - script_args_input, - requirements_input, - ml_framework_input, - message_or_button, - ) - - -def card_show_work(idx, request, state): - work = state["structures"]["ws"]["works"][f"w_{idx}"] - - def get_work_state(): - w = work["vars"].copy() - if "logs" in w: - w.pop("logs") - return pn.pane.JSON(w, theme="light", sizing_mode="stretch_both") - - options = { - "Expand to view your configuration": pn.pane.JSON( - request, theme="light", hover_preview=True, depth=4 - ), - "Expand to view logs": pn.Column( - pn.pane.Markdown( - "```bash\n" + to_str(work["vars"]["logs"]) + "\n```", - ), - height=800, - ), - "Expand to view your work state": get_work_state(), - } - selection_input = pn.widgets.RadioBoxGroup(name="Hello", options=list(options.keys())) - - @pn.depends(selection_input) - def selection_output(value): - return pn.panel(options[value], sizing_mode="stretch_both") - - return pn.Column( - selection_input, selection_output, sizing_mode="stretch_both", name=f"Run: {idx}" - ) - - -@pn.depends(app.param.state) -def view_run_lists_page(state: AppState): - title = "# Run Lists 🎈" - # Todo: Consider other layout than accordion. Don't think its that great - layout = pn.Accordion(sizing_mode="stretch_both") - print(state._request_state) - for idx, request in enumerate(state.requests): - layout.append(card_show_work(idx, request, state._state)) - layout.append(card_show_work(idx, request, state._state)) - return pn.Column(title, layout) - - -@pn.depends(app.param.state) -def app_state_page(state: AppState): - title = "# App State 🎈" - # Todo: Make this on stretch full heigh of its parent containe - json_output = pn.pane.JSON(state._state, theme="light", depth=6, max_height=800) - return pn.Column(title, json_output, scroll=True) - - -pn.Tabs( - ("New Run", create_new_page), - ("View your Runs", view_run_lists_page), - ("App State", app_state_page), - sizing_mode="stretch_both", -).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py new file mode 100644 index 0000000000000..dcf5a02f1fbb3 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py @@ -0,0 +1,249 @@ +import json +import os +from functools import partial +from unittest.mock import Mock + +import panel as pn +import param +from app_state_watcher import AppStateWatcher + +from lightning_app.utilities.state import AppState + + +def to_str(value): + if isinstance(value, list): + return "\n".join([str(item) for item in value]) + return str(value) + +if "LIGHTNING_FLOW_NAME" in os.environ: + app = AppStateWatcher() +else: + + class AppMock(param.Parameterized): + state = param.Parameter() + + app = AppMock(state=Mock()) + + + with open("state.json", "r") as fp: + app.state._state = json.load(fp) + app.state.requests = [ + { + "id": 0, + "train": { + "github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", + "script_path": "train_script.py", + "script_args": [ + "--trainer.max_epochs=5", + "--trainer.limit_train_batches=4", + "--trainer.limit_val_batches=4", + "--trainer.callbacks=ModelCheckpoint", + "--trainer.callbacks.monitor=val_acc", + ], + "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], + "ml_framework": "PyTorch Lightning", + }, + } + ] + + +ACCENT = "#792EE5" + +pn.config.raw_css.append( + """ + .bk-root { + height: calc( 100vh - 200px ) !important; + } + .state-container { + height: calc(100vh - 300px) !important; + } + .log-container { + height: calc(100vh - 380px) !important; + } + .scrollable { + overflow-x: hidden !important; + overflow-y: scroll !important; + } + """ +) + +pn.extension("terminal", sizing_mode="stretch_width", template="fast", notifications=True) +pn.state.template.param.update(site="Panel Lightning ⚡", title="Github Model Runner", accent_base_color=ACCENT, header_background=ACCENT) +pn.pane.JSON.param.hover_preview.default = True + +#region: Panel extensions + +def _to_value(value): + if hasattr(value, "value"): + return value.value + return value + +def bind_as_form(function, *args, submit, watch=False, **kwargs): + """Extends pn.bind to support "Forms" like binding. I.e. triggering only when a Submit button is clicked, + but using the dynamic values of widgets or Parameters as inputs. + + Args: + function (_type_): The function to execute + submit (_type_): The Submit widget or parameter to depend on + watch (bool, optional): Defaults to False. + + Returns: + _type_: A Reactive Function + """ + if not args: + args = [] + if not kwargs: + kwargs = {} + + def function_wrapper(_, args=args, kwargs=kwargs): + args=[_to_value[value] for value in args] + kwargs={key: _to_value(value) for key, value in kwargs.items()} + return function(*args, **kwargs) + return pn.bind(function_wrapper, submit, watch=watch) + +def show_value(widget): + """Shows the value of the widget or Parameter in a Panel + + Dynamically updated when ever the value changes + """ + def show(value): + return pn.panel(value, sizing_mode="stretch_both") + + return pn.bind(show, value=widget) + +THEME = pn.state.session_args.get("theme", [b"default"])[0].decode() +pn.pane.JSON.param.theme.default = THEME if THEME=="dark" else "light" + + +#endregion: Panel extensions +#region: Create new run + +def create_new_run(id, github_repo, script_path, script_args, requirements, ml_framework): + new_request = { + "id": id, + "train": { + "github_repo": github_repo, + "script_path": script_path, + "script_args": script_args.split(" "), + "requirements": requirements.split(" "), + "ml_framework": ml_framework, + }, + } + app.state.requests = app.state.requests + [new_request] + pn.state.notifications.send("New run created", background=ACCENT, icon='⚡') + +def message_or_button(ml_framework, submit_button): + if ml_framework not in ("PyTorch Lightning"): + return f"💥 {ml_framework} isn't supported yet." + else: + return submit_button + + +def create_new_page(): + id_input = pn.widgets.TextInput(name="Name your run", value="my_first_run") + github_repo_input = pn.widgets.TextInput( + name="Enter a Github Repo URL", + value="https://github.com/Lightning-AI/lightning-quick-start.git", + ) + script_path_input = pn.widgets.TextInput(name="Enter your script to run", value="train_script.py") + + script_args_input = pn.widgets.TextInput( + name="Enter your base script arguments", + value=( + "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 " + " --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" + ), + ) + requirements_input = pn.widgets.TextInput( + name="Enter your requirements", value="torchvision, pytorch_lightning, jsonargparse[signatures]" + ) + ml_framework_input = pn.widgets.RadioBoxGroup( + name="Select your ML Training Frameworks", + options=["PyTorch Lightning", "Keras", "Tensorflow"], + inline=True, + ) + submit_button = pn.widgets.Button(name="⚡ SUBMIT ⚡", button_type="primary") + + bind_as_form( + create_new_run, + id=id_input, + github_repo=github_repo_input, + script_path=script_path_input, + script_args=script_args_input, + requirements=requirements_input, + ml_framework=ml_framework_input, + submit=submit_button, + watch=True + ) + + return pn.Column( + "# Create a new run 🎈", + id_input, + github_repo_input, + script_path_input, + script_args_input, + requirements_input, + ml_framework_input, + pn.bind(partial(message_or_button, submit_button=submit_button), ml_framework_input), + ) + +#endregion: Create new run +#region: Run list page + +def configuration_component(request): + return pn.pane.JSON(request, depth=4) + +def work_state_ex_logs_component(work): + w = work["vars"].copy() + if "logs" in w: + w.pop("logs") + return pn.pane.JSON(w, depth=4) + +def log_component(work): + return pn.Column( + pn.pane.Markdown( + "```bash\n" + to_str(work["vars"]["logs"]) + "\n```", max_height=500 + ), scroll=True, css_classes=["log-container"] + ) + +def run_component(idx, request, state): + work = state["structures"]["ws"]["works"][f"w_{idx}"] + name=work["vars"]["id"] + return pn.Tabs( + ("Configuration", configuration_component(request)), + ("Work state", work_state_ex_logs_component(work)), + ("Logs", log_component(work)), + name=f"Run {idx}: {name}", margin=(5,0,0,0) + ) + + +@pn.depends(app.param.state) +def view_run_list_page(state: AppState): + title = "# View your runs 🎈" + layout = pn.Tabs(sizing_mode="stretch_both") + for idx, request in enumerate(state.requests): + layout.append(run_component(idx, request, state._state)) + # We just add some more dummy runs + layout.append(run_component(idx, request, state._state)) + layout.append(run_component(idx, request, state._state)) + return pn.Column(title, layout) + +#endregion: Run list page +#region: App state page + + +@pn.depends(app.param.state) +def view_app_state_page(state: AppState): + title = "# View the full state of the app 🎈" + json_output = pn.pane.JSON(state._state, depth=6) + return pn.Column(title, pn.Column(json_output, scroll=True, css_classes=["state-container"])) + +#endregion: App state page +#region: App +pn.Tabs( + ("New Run", create_new_page), + ("View Runs", view_run_list_page), + ("View State", view_app_state_page), + sizing_mode="stretch_both", +).servable() +#endregion: App From 75e482ae91967ba009fdda8d61479bffc6da7acf Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 06:46:42 +0200 Subject: [PATCH 020/103] fix issue --- .../add_web_ui/panel/examples/app_github_runner.py | 1 - .../add_web_ui/panel/examples/panel_github_runner.py | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py index 72c200ad7647c..18be9a201f6c9 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py @@ -12,7 +12,6 @@ from lightning.app.components.python import TracerPythonScript from panel_frontend import PanelFrontend from lightning.app.storage.path import Path -from app_state_watcher import AppStateWatcher class GithubRepoRunner(TracerPythonScript): def __init__( diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py index dcf5a02f1fbb3..32cbc4aee6782 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py @@ -151,7 +151,7 @@ def create_new_page(): name="Enter your base script arguments", value=( "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 " - " --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" + "--trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" ), ) requirements_input = pn.widgets.TextInput( @@ -163,7 +163,6 @@ def create_new_page(): inline=True, ) submit_button = pn.widgets.Button(name="⚡ SUBMIT ⚡", button_type="primary") - bind_as_form( create_new_run, id=id_input, @@ -222,10 +221,7 @@ def view_run_list_page(state: AppState): title = "# View your runs 🎈" layout = pn.Tabs(sizing_mode="stretch_both") for idx, request in enumerate(state.requests): - layout.append(run_component(idx, request, state._state)) - # We just add some more dummy runs - layout.append(run_component(idx, request, state._state)) - layout.append(run_component(idx, request, state._state)) + layout.append(run_component(idx, request, state._state)) return pn.Column(title, layout) #endregion: Run list page From 2ae8e873e8c0967823cbbb4500f71c9adfacf0f3 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 08:37:56 +0200 Subject: [PATCH 021/103] add basic docs --- .../workflows/add_web_ui/panel/basic.rst | 295 ++++++++++++++++++ .../add_web_ui/panel/examples/app_basic.py | 12 +- .../panel/examples/panel_app_basic.py | 50 +++ .../panel/examples/panel_frontend.py | 36 ++- .../frontend/panel/panel_frontend.py | 39 ++- 5 files changed, 416 insertions(+), 16 deletions(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/basic.rst create mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst new file mode 100644 index 0000000000000..0976faa51913e --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -0,0 +1,295 @@ +################################### +Add a web UI with Panel (basic) +################################### + +**Audience:** Users who want to add a web UI written with Python. + +**Prereqs:** Basic Python knowledge. + +---- + +************** +What is Panel? +************** + +`Panel`_ and the `HoloViz`_ ecosystem provides unique and powerful +features such as big data viz via `DataShader`_, easy cross filtering +via `HoloViews`_, streaming and much more. + +- Panel works with the tools you know and love ❤️. Panel ties into the PyData and Jupyter ecosystems as you can develop in notebooks and use ipywidgets. You can also develop in .py files. +- Panel is one of the 4 most popular data app frameworks in Python with `more than 400.000 downloads a month `_. It's especially popular in the scientific community. +- Panel is used by for example Rapids to power `CuxFilter`_, a CuDF based big data viz framework. +- Panel can be deployed on your favorite server or cloud including `Lightning`_. + +Panel is **particularly well suited for lightning.ai apps** that needs to display live progress from +`LightningWork` as the Panel server can react to progress and asynchronously push messages from the server to the +client via web socket communication. + +Install Panel with: + +.. code:: bash + + pip install panel + +---- + +************************* +Run a basic Panel app +************************* + +In the next few sections we'll build an app step-by-step. + +First **create a file named `panel_app_basic.py`** with the app content: + +.. code:: python + + import panel as pn + + pn.panel("Hello **Panel ⚡** World").servable() + +Then **create a file named `app_basic.py`** with the app content: + +.. code:: python + + import lightning as L + from lightning_app.frontend.panel import PanelFrontend + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("panel_app_basic.py") + + def configure_layout(self): + return self._frontend + + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + + app = L.LightningApp(LitApp()) + +add "Panel" to a requirements.txt file: + +.. code:: bash + + echo 'panel' >> requirements.txt + +this is a best practice to make apps reproducible. + +---- + +*********** +Run the app +*********** + +Run the app locally to see it! + +.. code:: bash + + lightning run app app_basic.py + +.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/images/panel-lightning/panel-lightning-basic.png + :alt: Basic Panel Lightning App + + Basic Panel Lightning App + +Now run it on the cloud as well: + +.. code:: bash + + lightning run app app_basic.py --cloud + +---- + +************************ +Step-by-step walkthrough +************************ + +In this section, we explain each part of this code in detail. + +---- + +0. Define a Panel app +^^^^^^^^^^^^^^^^^^^^^^^^^ + +First, find the Panel app you want to integrate. In this example, that app looks like: + +.. code:: python + + import panel as pn + + pn.panel("Hello **Panel ⚡** World").servable() + +Refer to the `Panel documentation `_ for more complex examples. + +---- + +1. Add Panel to a component +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Link this app to the Lightning App by using the ``PanelFrontend`` class which needs to be returned from +the ``configure_layout`` method of the Lightning component you want to connect to Panel. + +.. code:: python + :emphasize-lines: 7,10 + + import lightning as L + from lightning_app.frontend.panel import PanelFrontend + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("panel_app_basic.py") + + def configure_layout(self): + return self._frontend + + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + + app = L.LightningApp(LitApp()) + +The argument of the ``PanelFrontend`` class, points to the script, notebook or function that +runs your Panel app. + +---- + +1. Route the UI in the root component +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The second step, is to tell the Root component in which tab to render this component's UI. +In this case, we render the ``LitPanel`` UI in the ``home`` tab of the application. + +.. code:: python + :emphasize-lines: 18 + + import lightning as L + from lightning_app.frontend.panel import PanelFrontend + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("panel_app_basic.py") + + def configure_layout(self): + return self._frontend + + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + +2. Autoreload +^^^^^^^^^^^^^ + +You can run your lightning app with Panel **autoreload** by setting the environment variable +``PANEL_AUTORELOAD`` to ``yes``. + +.. code-block:: + + PANEL_AUTORELOAD=yes lightning run app app_basic.py + +.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-autoreload.gif + :alt: Basic Panel Lightning App + + Basic Panel Lightning App with autoreload + +3. Theming +^^^^^^^^^^ + +To theme your app you, can use the lightning accent color #792EE5 with the `FastListTemplate`_. + +Try replacing the contents of `app_basic.py` with the below code. + +.. code:: bash + + import panel as pn + import plotly.express as px + + ACCENT = "#792EE5" + + pn.extension("plotly", sizing_mode="stretch_width", template="fast") + pn.state.template.param.update( + title="⚡ Hello Panel + Lightning ⚡", accent_base_color=ACCENT, header_background=ACCENT + ) + + pn.config.raw_css.append( + """ + .bk-root:first-of-type { + height: calc( 100vh - 200px ) !important; + } + """ + ) + + + def get_panel_theme(): + """Returns 'default' or 'dark'""" + return pn.state.session_args.get("theme", [b"default"])[0].decode() + + + def get_plotly_template(): + if get_panel_theme() == "dark": + return "plotly_dark" + return "plotly_white" + + + def get_plot(length=5): + xseries = [index for index in range(length + 1)] + yseries = [x**2 for x in xseries] + fig = px.line( + x=xseries, + y=yseries, + template=get_plotly_template(), + color_discrete_sequence=[ACCENT], + range_x=(0, 10), + markers=True, + ) + fig.layout.autosize = True + return fig + + + length = pn.widgets.IntSlider(value=5, start=1, end=10, name="Length") + dynamic_plot = pn.panel( + pn.bind(get_plot, length=length), sizing_mode="stretch_both", config={"responsive": True} + ) + pn.Column(length, dynamic_plot).servable() + +Run `pip install plotly pandas` and remember to add the dependencies to the requirements.txt file: + +.. code:: bash + + echo 'plotly' >> requirements.txt + echo 'pandas' >> requirements.txt + +Finally run the app + +.. code:: bash + + lightning run app app_basic.py + +.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-theme.gif + :alt: Basic Panel Lightning App + + Basic Panel Plotly Lightning App with theming + +.. _Panel: https://panel.holoviz.org/ +.. _FastListTemplate: https://panel.holoviz.org/reference/templates/FastListTemplate.html#templates-gallery-fastlisttemplate +.. _HoloViz: https://holoviz.org/ +.. _DataShader: https://datashader.org/ +.. _HoloViews: https://holoviews.org/ +.. _Lightning: https://lightning.ai/ +.. _CuxFilter: https://github.com/rapidsai/cuxfilter \ No newline at end of file diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py index e23efeacdcc52..61b598ea8e22f 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py @@ -1,24 +1,14 @@ -# app.py -import panel as pn - import lightning as L -# Todo: change import # from lightning_app.frontend.panel import PanelFrontend from panel_frontend import PanelFrontend - -def your_panel_app(app): - pn.pane.Markdown("hello").servable() - class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend(your_panel_app) + self._frontend = PanelFrontend("panel_app_basic.py") def configure_layout(self): return self._frontend - - class LitApp(L.LightningFlow): def __init__(self): super().__init__() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py new file mode 100644 index 0000000000000..0881712bc10c5 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py @@ -0,0 +1,50 @@ +import panel as pn +import plotly.express as px + +ACCENT = "#792EE5" + +pn.extension("plotly", sizing_mode="stretch_width", template="fast") +pn.state.template.param.update( + title="⚡ Hello Panel + Lightning ⚡", accent_base_color=ACCENT, header_background=ACCENT +) + +pn.config.raw_css.append( + """ + .bk-root:first-of-type { + height: calc( 100vh - 150px ) !important; + } + """ +) + + +def get_panel_theme(): + """Returns 'default' or 'dark'""" + return pn.state.session_args.get("theme", [b"default"])[0].decode() + + +def get_plotly_template(): + if get_panel_theme() == "dark": + return "plotly_dark" + return "plotly_white" + + +def get_plot(length=5): + xseries = [index for index in range(length + 1)] + yseries = [x**2 for x in xseries] + fig = px.line( + x=xseries, + y=yseries, + template=get_plotly_template(), + color_discrete_sequence=[ACCENT], + range_x=(0, 10), + markers=True, + ) + fig.layout.autosize = True + return fig + + +length = pn.widgets.IntSlider(value=5, start=1, end=10, name="Length") +dynamic_plot = pn.panel( + pn.bind(get_plot, length=length), sizing_mode="stretch_both", config={"responsive": True} +) +pn.Column(length, dynamic_plot).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py index 7120d811be994..7d94b13459314 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -18,7 +18,6 @@ class PanelFrontend(Frontend): - # Todo: Add Example to docstring """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. To use this frontend, you must first install the `panel` package: @@ -27,6 +26,41 @@ class PanelFrontend(Frontend): pip install panel + Example: + + `panel_app_basic.py` + + .. code-block:: python + + import panel as pn + + pn.panel("Hello **Panel ⚡** World").servable() + + `app_basic.py` + + .. code-block:: python + + import lightning as L + from lightning_app.frontend.panel import PanelFrontend + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("panel_app_basic.py") + + def configure_layout(self): + return self._frontend + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + + app = L.LightningApp(LitApp()) + Please note the Panel server will be logging output to error.log and output.log files respectively. diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index e18b7f65ae823..abe3d67bcac0a 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -22,7 +22,6 @@ class PanelFrontend(Frontend): - # Todo: Add Example to docstring """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. To use this frontend, you must first install the `panel` package: @@ -31,11 +30,43 @@ class PanelFrontend(Frontend): pip install panel - Please note the Panel server will be logging output to error.log and output.log files - respectively. + Example: + + `panel_app_basic.py` + + .. code-block:: python + + import panel as pn + + pn.panel("Hello **Panel ⚡** World").servable() + + `app_basic.py` + + .. code-block:: python + + import lightning as L + from lightning_app.frontend.panel import PanelFrontend + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("panel_app_basic.py") + + def configure_layout(self): + return self._frontend + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + + app = L.LightningApp(LitApp()) You can start the lightning server with Panel autoreload by setting the `PANEL_AUTORELOAD` - environment variable to 'yes': `AUTORELOAD=yes lightning run app my_app.py`. + environment variable to 'yes': `AUTORELOAD=yes lightning run app app_basic.py`. Args: render_fn_or_file: A pure function or the path to a .py or .ipynb file. From 4fd2cf1c84c491ab19f22dd76fc8e77b1a704949 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 08:42:02 +0200 Subject: [PATCH 022/103] fix errors --- .../workflows/add_web_ui/panel/basic.rst | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 0976faa51913e..caa56d882a647 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -39,7 +39,7 @@ Run a basic Panel app In the next few sections we'll build an app step-by-step. -First **create a file named `panel_app_basic.py`** with the app content: +First **create a file named ``panel_app_basic.py``** with the app content: .. code:: python @@ -47,7 +47,7 @@ First **create a file named `panel_app_basic.py`** with the app content: pn.panel("Hello **Panel ⚡** World").servable() -Then **create a file named `app_basic.py`** with the app content: +Then **create a file named ``app_basic.py``** with the app content: .. code:: python @@ -73,7 +73,7 @@ Then **create a file named `app_basic.py`** with the app content: app = L.LightningApp(LitApp()) -add "Panel" to a requirements.txt file: +add "panel" to a requirements.txt file: .. code:: bash @@ -165,7 +165,7 @@ runs your Panel app. ---- -1. Route the UI in the root component +2. Route the UI in the root component ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The second step, is to tell the Root component in which tab to render this component's UI. @@ -193,8 +193,9 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati def configure_layout(self): return {"name": "home", "content": self.lit_panel} -2. Autoreload -^^^^^^^^^^^^^ +********** +Autoreload +********** You can run your lightning app with Panel **autoreload** by setting the environment variable ``PANEL_AUTORELOAD`` to ``yes``. @@ -208,12 +209,13 @@ You can run your lightning app with Panel **autoreload** by setting the environm Basic Panel Lightning App with autoreload -3. Theming -^^^^^^^^^^ +******* +Theming +******* To theme your app you, can use the lightning accent color #792EE5 with the `FastListTemplate`_. -Try replacing the contents of `app_basic.py` with the below code. +Try replacing the contents of ``app_basic.py`` with the below code. .. code:: bash From 2190b63069efaff0bdb9010a3ff89a06774c9c83 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 08:52:04 +0200 Subject: [PATCH 023/103] add panel intro --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index caa56d882a647..5ed232e8e0399 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -21,6 +21,11 @@ via `HoloViews`_, streaming and much more. - Panel is used by for example Rapids to power `CuxFilter`_, a CuDF based big data viz framework. - Panel can be deployed on your favorite server or cloud including `Lightning`_. +.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-intro.gif + :alt: Basic Panel Lightning App + + Example Panel App + Panel is **particularly well suited for lightning.ai apps** that needs to display live progress from `LightningWork` as the Panel server can react to progress and asynchronously push messages from the server to the client via web socket communication. @@ -125,7 +130,7 @@ First, find the Panel app you want to integrate. In this example, that app looks pn.panel("Hello **Panel ⚡** World").servable() -Refer to the `Panel documentation `_ for more complex examples. +Refer to the `Panel documentation `_ or `awesome-panel.org `_ for more complex examples. ---- @@ -294,4 +299,5 @@ Finally run the app .. _DataShader: https://datashader.org/ .. _HoloViews: https://holoviews.org/ .. _Lightning: https://lightning.ai/ -.. _CuxFilter: https://github.com/rapidsai/cuxfilter \ No newline at end of file +.. _CuxFilter: https://github.com/rapidsai/cuxfilter +.. _AwesomePanel: https://awesome-panel.org/home From ed3fbeb314f5f509069156c2fa4ed60465bad98e Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 09:00:03 +0200 Subject: [PATCH 024/103] add more gifs --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 5ed232e8e0399..8a7494549d76c 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -22,7 +22,7 @@ via `HoloViews`_, streaming and much more. - Panel can be deployed on your favorite server or cloud including `Lightning`_. .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-intro.gif - :alt: Basic Panel Lightning App + :alt: Example Panel App Example Panel App @@ -30,6 +30,11 @@ Panel is **particularly well suited for lightning.ai apps** that needs to displa `LightningWork` as the Panel server can react to progress and asynchronously push messages from the server to the client via web socket communication. +.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-intro.gif + :alt: Example Panel Streaming App + + Example Panel Streaming App + Install Panel with: .. code:: bash @@ -210,7 +215,7 @@ You can run your lightning app with Panel **autoreload** by setting the environm PANEL_AUTORELOAD=yes lightning run app app_basic.py .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-autoreload.gif - :alt: Basic Panel Lightning App + :alt: Basic Panel Lightning App with autoreload Basic Panel Lightning App with autoreload @@ -289,7 +294,7 @@ Finally run the app lightning run app app_basic.py .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-theme.gif - :alt: Basic Panel Lightning App + :alt: Basic Panel Plotly Lightning App with theming Basic Panel Plotly Lightning App with theming From fc5e2c52c42d06e0442a91877046bbc350834682 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 09:01:33 +0200 Subject: [PATCH 025/103] fix link --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 8a7494549d76c..4b005589f0bc2 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -30,7 +30,7 @@ Panel is **particularly well suited for lightning.ai apps** that needs to displa `LightningWork` as the Panel server can react to progress and asynchronously push messages from the server to the client via web socket communication. -.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-intro.gif +.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-streaming.gif :alt: Example Panel Streaming App Example Panel Streaming App From 7b28e9d1b6ad3d3ea7d5d470364e3f5bec6c9dc3 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 09:08:24 +0200 Subject: [PATCH 026/103] git push --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 4b005589f0bc2..3ad33ba838faf 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -207,8 +207,10 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati Autoreload ********** -You can run your lightning app with Panel **autoreload** by setting the environment variable -``PANEL_AUTORELOAD`` to ``yes``. +To speed up your development workflow, you can run your lightning app with Panel **autoreload** by +setting the environment variable ``PANEL_AUTORELOAD`` to ``yes``. + +Try running the below .. code-block:: From 30110e4d6d661f17dbfc07eae320e489a018732a Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 09:11:25 +0200 Subject: [PATCH 027/103] simplify --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 3ad33ba838faf..47e2c8a8419c3 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -26,9 +26,8 @@ via `HoloViews`_, streaming and much more. Example Panel App -Panel is **particularly well suited for lightning.ai apps** that needs to display live progress from -`LightningWork` as the Panel server can react to progress and asynchronously push messages from the server to the -client via web socket communication. +Panel is **particularly well suited for lightning.ai apps** that needs to display live progress as the Panel server can react +to progress and asynchronously push messages from the server to the client via web socket communication. .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-streaming.gif :alt: Example Panel Streaming App From 7329a498ab04ac26d69bad247cddcfd4509314ed Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 09:12:46 +0200 Subject: [PATCH 028/103] add more docs --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 47e2c8a8419c3..70e2ddcbb51ca 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -102,6 +102,8 @@ Run the app locally to see it! lightning run app app_basic.py +The app should look like the below + .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/images/panel-lightning/panel-lightning-basic.png :alt: Basic Panel Lightning App @@ -115,6 +117,8 @@ Now run it on the cloud as well: ---- +Todo: Insert figure of app running in cloud + ************************ Step-by-step walkthrough ************************ From b75bdad664c423f43687b66c16fdb296e94fd8c0 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 11:47:59 +0200 Subject: [PATCH 029/103] change app_basic to app --- .../workflows/add_web_ui/panel/basic.rst | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 70e2ddcbb51ca..5aee2581a5aa6 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -48,7 +48,7 @@ Run a basic Panel app In the next few sections we'll build an app step-by-step. -First **create a file named ``panel_app_basic.py``** with the app content: +First **create a file named ``panel_app.py``** with the app content: .. code:: python @@ -56,7 +56,7 @@ First **create a file named ``panel_app_basic.py``** with the app content: pn.panel("Hello **Panel ⚡** World").servable() -Then **create a file named ``app_basic.py``** with the app content: +Then **create a file named ``app.py``** with the app content: .. code:: python @@ -66,7 +66,7 @@ Then **create a file named ``app_basic.py``** with the app content: class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend("panel_app_basic.py") + self._frontend = PanelFrontend("panel_app.py") def configure_layout(self): return self._frontend @@ -100,7 +100,7 @@ Run the app locally to see it! .. code:: bash - lightning run app app_basic.py + lightning run app app.py The app should look like the below @@ -113,7 +113,7 @@ Now run it on the cloud as well: .. code:: bash - lightning run app app_basic.py --cloud + lightning run app app.py --cloud ---- @@ -157,7 +157,7 @@ the ``configure_layout`` method of the Lightning component you want to connect t class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend("panel_app_basic.py") + self._frontend = PanelFrontend("panel_app.py") def configure_layout(self): return self._frontend @@ -193,7 +193,7 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend("panel_app_basic.py") + self._frontend = PanelFrontend("panel_app.py") def configure_layout(self): return self._frontend @@ -217,7 +217,7 @@ Try running the below .. code-block:: - PANEL_AUTORELOAD=yes lightning run app app_basic.py + PANEL_AUTORELOAD=yes lightning run app app.py .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-autoreload.gif :alt: Basic Panel Lightning App with autoreload @@ -230,7 +230,7 @@ Theming To theme your app you, can use the lightning accent color #792EE5 with the `FastListTemplate`_. -Try replacing the contents of ``app_basic.py`` with the below code. +Try replacing the contents of ``app.py`` with the below code. .. code:: bash @@ -296,7 +296,7 @@ Finally run the app .. code:: bash - lightning run app app_basic.py + lightning run app app.py .. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-theme.gif :alt: Basic Panel Plotly Lightning App with theming From 8576e121ad3a0a9cd90c80a632b6bb9003d44817 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 11:53:10 +0200 Subject: [PATCH 030/103] add panel to documentation --- .../workflows/add_web_ui/index_content.rst | 1 + .../workflows/add_web_ui/panel/index.rst | 83 +++++++++++++++++++ .../intermediate.rst | 3 +- 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 docs/source-app/workflows/add_web_ui/panel/index.rst diff --git a/docs/source-app/workflows/add_web_ui/index_content.rst b/docs/source-app/workflows/add_web_ui/index_content.rst index 6995d0e9b2768..6ea749395575b 100644 --- a/docs/source-app/workflows/add_web_ui/index_content.rst +++ b/docs/source-app/workflows/add_web_ui/index_content.rst @@ -4,6 +4,7 @@ dash/index gradio/index + panel/index streamlit/index .. toctree:: diff --git a/docs/source-app/workflows/add_web_ui/panel/index.rst b/docs/source-app/workflows/add_web_ui/panel/index.rst new file mode 100644 index 0000000000000..380e3318292e2 --- /dev/null +++ b/docs/source-app/workflows/add_web_ui/panel/index.rst @@ -0,0 +1,83 @@ +.. toctree:: + :maxdepth: 1 + :hidden: + + basic + intermediate + +####################### +Add a web UI with Panel +####################### + +.. raw:: html + +
+
+ +.. Add callout items below this line + +.. displayitem:: + :header: 1: Connect Panel + :description: Learn how to connect Panel to a Lightning Component. + :col_css: col-md-6 + :button_link: basic.html + :height: 150 + :tag: basic + +.. displayitem:: + :header: 2: Enable two-way communication + :description: Enable two-way communication between Panel and a Lightning App. + :col_css: col-md-6 + :button_link: intermediate.html + :height: 150 + :tag: intermediate + +.. raw:: html + +
+
+ +---- + +******** +Examples +******** + +Here are a few example apps that use a Panel web UI. + + +.. raw:: html + +
+
+ +.. Add callout items below this line + +.. displayitem:: + :header: Example 1 + :description: Show off your work! Contribute an example. + :col_css: col-md-4 + :button_link: ../../../contribute_app.html + :height: 150 + :tag: Waiting for contributed example + +.. displayitem:: + :header: Example 2 + :description: Show off your work! Contribute an example. + :col_css: col-md-4 + :button_link: ../../../contribute_app.html + :height: 150 + :tag: Waiting for contributed example + +.. displayitem:: + :header: Example 3 + :description: Show off your work! Contribute an example. + :col_css: col-md-4 + :button_link: ../../../contribute_app.html + :height: 150 + :tag: Waiting for contributed example + +.. raw:: html + +
+
diff --git a/docs/source-app/workflows/build_lightning_component/intermediate.rst b/docs/source-app/workflows/build_lightning_component/intermediate.rst index a1956b260df9b..c4ac53bd83fd5 100644 --- a/docs/source-app/workflows/build_lightning_component/intermediate.rst +++ b/docs/source-app/workflows/build_lightning_component/intermediate.rst @@ -9,7 +9,8 @@ Build a Lightning component (intermediate) Add a web user interface (UI) ***************************** Every lightning component can have its own user interface (UI). Lightning components support any kind -of UI interface such as react.js, vue.js, streamlit, gradio, dash, web urls, etc...(`full list here <../add_web_ui/index.html>`_). +of UI interface such as dash, gradio, panel, react.js, streamlit, vue.js, web urls, +etc...(`full list here <../add_web_ui/index.html>`_). Let's say that we have a user interface defined in html: From a2700bc9c582fdab4fb1b6ddc5f715c9c0ab2050 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 12:44:25 +0200 Subject: [PATCH 031/103] update docs --- .../workflows/add_web_ui/panel/basic.rst | 18 +- .../add_web_ui/panel/intermediate.rst | 158 +++++++++++++++++- 2 files changed, 164 insertions(+), 12 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 5aee2581a5aa6..47a7248f02372 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -2,7 +2,7 @@ Add a web UI with Panel (basic) ################################### -**Audience:** Users who want to add a web UI written with Python. +**Audience:** Users who want to add a web UI written with Python and Panel. **Prereqs:** Basic Python knowledge. @@ -48,10 +48,12 @@ Run a basic Panel app In the next few sections we'll build an app step-by-step. -First **create a file named ``panel_app.py``** with the app content: +First **create a file named ``app_panel.py``** with the app content: .. code:: python + # app_panel.py + import panel as pn pn.panel("Hello **Panel ⚡** World").servable() @@ -60,13 +62,15 @@ Then **create a file named ``app.py``** with the app content: .. code:: python + # app.py + import lightning as L from lightning_app.frontend.panel import PanelFrontend class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend("panel_app.py") + self._frontend = PanelFrontend("app_panel.py") def configure_layout(self): return self._frontend @@ -157,7 +161,7 @@ the ``configure_layout`` method of the Lightning component you want to connect t class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend("panel_app.py") + self._frontend = PanelFrontend("app_panel.py") def configure_layout(self): return self._frontend @@ -193,7 +197,7 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati class LitPanel(L.LightningFlow): def __init__(self): super().__init__() - self._frontend = PanelFrontend("panel_app.py") + self._frontend = PanelFrontend("app_panel.py") def configure_layout(self): return self._frontend @@ -230,10 +234,12 @@ Theming To theme your app you, can use the lightning accent color #792EE5 with the `FastListTemplate`_. -Try replacing the contents of ``app.py`` with the below code. +Try replacing the contents of ``app_panel.py`` with the below code. .. code:: bash + # app_panel.py + import panel as pn import plotly.express as px diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index 03fb9bb7bca23..716255d04baab 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -8,11 +8,157 @@ Add a web UI with Panel (intermediate) ---- -************************ -Display Global App State -************************ +************************************** +Interact with the component from Panel +************************************** -The `PanelFrontend` enables you read/pull and display (parts of) the global -app state. +The panel UI enables user interactions with the Lightning App via widgets. +You can modify the state variables of a Lightning component via the ``AppStateWatcher``. -See app_pull_state.py. +For example, here we increase the count variable of the Lightning Component every time a user +presses a button: + +.. code:: bash + + # app_panel.py + + import panel as pn + from lightning_app.frontend.panel import AppStateWatcher + + pn.extension(sizing_mode="stretch_width") + + app = AppStateWatcher() + + submit_button = pn.widgets.Button(name="submit") + + @pn.depends(submit_button, watch=True) + def submit(_): + app.state.count += 1 + + @pn.depends(app.param.state) + def current_count(_): + return f"current count: {app.state.count}" + + pn.Column( + submit_button, + current_count, + ).servable() + + + +.. code:: bash + + # app.py + + import lightning as L + from lightning_app.frontend.panel import PanelFrontend + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("app_panel.py") + self.count = 0 + self._last_count=0 + + def configure_layout(self): + return self._frontend + + def run(self): + if self.count != self._last_count: + self._last_count=self.count + print("Count changed to: ", self.count) + + + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def run(self): + self.lit_panel.run() + + def configure_layout(self): + return {"name": "home", "content": self.lit_panel} + + + app = L.LightningApp(LitApp()) + +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-counter-from-frontend.gif + :alt: Panel Lightning App updating a counter in the frontend + + Panel Lightning App updating a counter in the frontend + +---- + +************************************** +Interact with Panel from the component +************************************** + +To update the Panel UI from any Lightning component, update the property in the component. Make sure to call ``run`` method from the +parent component. + +In this example we update the value of the counter from the component: + +.. code:: bash + + # app_panel.py + + import panel as pn + from lightning_app.frontend.panel import AppStateWatcher + + app=AppStateWatcher() + + pn.extension(sizing_mode="stretch_width") + + def counter(state): + return f"Counter: {state.counter}" + + last_update = pn.bind(counter, app.param.state) + + pn.panel(last_update).servable() + +.. code:: bash + + # app.py + + from datetime import datetime as dt + from lightning_app.frontend.panel import PanelFrontend + + import lightning_app as L + + + class LitPanel(L.LightningFlow): + def __init__(self): + super().__init__() + self._frontend = PanelFrontend("app_panel.py") + self.counter = 0 + self._last_update=dt.now() + + def configure_layout(self): + return self._frontend + + def run(self): + now = dt.now() + if (now-self._last_update).microseconds>=250: + self.counter += 1 + self._last_update = now + print("Counter changed to: ", self.counter) + + class LitApp(L.LightningFlow): + def __init__(self): + super().__init__() + self.lit_panel = LitPanel() + + def run(self): + self.lit_panel.run() + + def configure_layout(self): + tab1 = {"name": "home", "content": self.lit_panel} + return tab1 + + app = L.LightningApp(LitApp()) + +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-counter-from-component.gif + :alt: Panel Lightning App updating a counter from the component + + Panel Lightning App updating a counter from the component \ No newline at end of file From 2fad50d291975bcba524bec86d2f1eed975de7ef Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 12:57:40 +0200 Subject: [PATCH 032/103] change counter to count --- .../add_web_ui/panel/intermediate.rst | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index 716255d04baab..2c8a3fd05045a 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -12,10 +12,10 @@ Add a web UI with Panel (intermediate) Interact with the component from Panel ************************************** -The panel UI enables user interactions with the Lightning App via widgets. +The ``PanelFrontend`` enables user interactions with the Lightning App via widgets. You can modify the state variables of a Lightning component via the ``AppStateWatcher``. -For example, here we increase the count variable of the Lightning Component every time a user +For example, here we increase the ``count`` variable of the Lightning Component every time a user presses a button: .. code:: bash @@ -84,9 +84,9 @@ presses a button: app = L.LightningApp(LitApp()) .. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-counter-from-frontend.gif - :alt: Panel Lightning App updating a counter in the frontend + :alt: Panel Lightning App updating a counter from the frontend - Panel Lightning App updating a counter in the frontend + Panel Lightning App updating a counter from the frontend ---- @@ -94,10 +94,10 @@ presses a button: Interact with Panel from the component ************************************** -To update the Panel UI from any Lightning component, update the property in the component. Make sure to call ``run`` method from the +To update the `PanelFrontend` from any Lightning component, update the property in the component. Make sure to call ``run`` method from the parent component. -In this example we update the value of the counter from the component: +In this example we update the value of ``count`` from the component: .. code:: bash @@ -111,7 +111,7 @@ In this example we update the value of the counter from the component: pn.extension(sizing_mode="stretch_width") def counter(state): - return f"Counter: {state.counter}" + return f"Counter: {state.count}" last_update = pn.bind(counter, app.param.state) @@ -131,7 +131,7 @@ In this example we update the value of the counter from the component: def __init__(self): super().__init__() self._frontend = PanelFrontend("app_panel.py") - self.counter = 0 + self.count = 0 self._last_update=dt.now() def configure_layout(self): @@ -140,9 +140,9 @@ In this example we update the value of the counter from the component: def run(self): now = dt.now() if (now-self._last_update).microseconds>=250: - self.counter += 1 + self.count += 1 self._last_update = now - print("Counter changed to: ", self.counter) + print("Counter changed to: ", self.count) class LitApp(L.LightningFlow): def __init__(self): From c424f21614bfd17191f3cf64ab4bb5565888ecfe Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Wed, 13 Jul 2022 13:27:01 +0200 Subject: [PATCH 033/103] add github runner visuals --- .../add_web_ui/panel/intermediate.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index 2c8a3fd05045a..21e4a3e1fcbf3 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -161,4 +161,21 @@ In this example we update the value of ``count`` from the component: .. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-counter-from-component.gif :alt: Panel Lightning App updating a counter from the component - Panel Lightning App updating a counter from the component \ No newline at end of file + Panel Lightning App updating a counter from the component + +******************* +Panel Tips & Tricks +******************* + +- Caching: Panel provides the easy to use ```pn.state.cache` memory based, ``dict`` caching. If you are looking for something persistent try `DiskCache `_ its really powerful and simple to use. You can use it to communicate large amounts of data between the components and frontend(s). +- Notifactions: Panel provides easy to use `notifications `_. You can for example use them to provide notifications about runs starting or ending. +- Tabulator Table: Panel provides the `Tabulator table `_ which features expandable rows. The table is useful to provide for example an overview of you runs. But you can dig into the details by clicking and expanding the row. +- Task Scheduling: Panel provides easy to use `task scheduling `. You can use this to for example read and display files created by your components on a schedule basis. +- Terminal: Panel provides the `Xterm.js terminal `_ which can be used to display live logs from your components and allow you to provide a terminal interface to your component. + +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-github-runner.gif + :alt: Panel Lightning App running models on github + + Panel Lightning App running models on github + +# Todo: Add link to the code and running app. \ No newline at end of file From f2f0910fabf10800a905b32f48892573231ba85d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:29:48 +0000 Subject: [PATCH 034/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../workflows/add_web_ui/panel/basic.rst | 6 ++ .../add_web_ui/panel/examples/app_basic.py | 6 +- .../panel/examples/app_basic_script.py | 3 +- .../panel/examples/app_github_runner.py | 26 +++--- .../examples/app_interact_from_component.py | 5 +- .../examples/app_interact_from_frontend.py | 7 +- .../panel/examples/app_state_comm.py | 3 +- .../panel/examples/app_state_watcher.py | 2 +- .../examples/app_streamlit_github_render.py | 25 +++--- .../add_web_ui/panel/examples/copy.sh | 2 +- .../add_web_ui/panel/examples/other.py | 9 +- .../panel/examples/panel_app_basic.py | 10 +-- .../panel/examples/panel_frontend.py | 15 ++-- .../panel/examples/panel_github_runner.py | 82 +++++++++++-------- .../panel/examples/panel_serve_render_fn.py | 1 - .../add_web_ui/panel/examples/state.json | 2 +- .../add_web_ui/panel/intermediate.rst | 6 +- .../add_web_ui/panel/tips_and_tricks.rst | 2 +- src/lightning_app/frontend/panel/__init__.py | 4 +- .../frontend/panel/panel_frontend.py | 13 +-- .../frontend/utilities/app_state_comm.py | 3 +- src/lightning_app/frontend/utilities/other.py | 9 +- tests/tests_app/frontend/conftest.py | 11 ++- tests/tests_app/frontend/panel/app_panel.py | 2 +- .../frontend/panel/test_panel_frontend.py | 16 ++-- .../panel/test_panel_serve_render_fn.py | 5 +- .../frontend/utilities/test_app_state_comm.py | 9 +- .../utilities/test_app_state_watcher.py | 6 +- .../frontend/utilities/test_other.py | 16 ++-- 29 files changed, 161 insertions(+), 145 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 47a7248f02372..ea3d312fa6879 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -67,6 +67,7 @@ Then **create a file named ``app.py``** with the app content: import lightning as L from lightning_app.frontend.panel import PanelFrontend + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() @@ -75,6 +76,7 @@ Then **create a file named ``app.py``** with the app content: def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -158,6 +160,7 @@ the ``configure_layout`` method of the Lightning component you want to connect t import lightning as L from lightning_app.frontend.panel import PanelFrontend + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() @@ -166,6 +169,7 @@ the ``configure_layout`` method of the Lightning component you want to connect t def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -194,6 +198,7 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati import lightning as L from lightning_app.frontend.panel import PanelFrontend + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() @@ -202,6 +207,7 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati def configure_layout(self): return self._frontend + class LitApp(L.LightningFlow): def __init__(self): super().__init__() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py index 61b598ea8e22f..9f41f095d66a9 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py @@ -1,7 +1,9 @@ -import lightning as L # from lightning_app.frontend.panel import PanelFrontend from panel_frontend import PanelFrontend +import lightning as L + + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() @@ -9,6 +11,8 @@ def __init__(self): def configure_layout(self): return self._frontend + + class LitApp(L.LightningFlow): def __init__(self): super().__init__() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py index 2d275cb12f728..3d427a1ce6062 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py @@ -1,11 +1,12 @@ # app.py import panel as pn -import lightning as L # Todo: change import # from lightning_app.frontend.panel import PanelFrontend from panel_frontend import PanelFrontend +import lightning as L + class LitPanel(L.LightningFlow): def __init__(self): diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py index 18be9a201f6c9..e5bfc87f32104 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py @@ -7,12 +7,14 @@ from subprocess import Popen from typing import Dict, List, Optional +from panel_frontend import PanelFrontend + from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow from lightning.app import structures from lightning.app.components.python import TracerPythonScript -from panel_frontend import PanelFrontend from lightning.app.storage.path import Path + class GithubRepoRunner(TracerPythonScript): def __init__( self, @@ -24,8 +26,8 @@ def __init__( cloud_compute: Optional[CloudCompute] = None, **kwargs, ): - """The GithubRepoRunner Component clones a repo, - runs a specific script with provided arguments and collect logs. + """The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect + logs. Arguments: id: Identified of the component. @@ -41,7 +43,7 @@ def __init__( cloud_compute=cloud_compute, cloud_build_config=BuildConfig(requirements=requirements), ) - self.script_path=script_path + self.script_path = script_path self.id = id self.github_repo = github_repo self.kwargs = kwargs @@ -55,8 +57,7 @@ def run(self, *args, **kwargs): # 2: Use git command line to clone the repo. repo_name = self.github_repo.split("/")[-1].replace(".git", "") cwd = os.path.dirname(__file__) - subprocess.Popen( - f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() + subprocess.Popen(f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() # 3: Execute the parent run method of the TracerPythonScript class. os.chdir(os.path.join(cwd, repo_name)) @@ -72,7 +73,6 @@ def configure_layout(self): class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.best_model_path = None @@ -104,12 +104,12 @@ def trainer_pre_fn(self, *args, work=None, **kwargs): # 5. Patch the `__init__` method of the Trainer # to inject our callback with a reference to the work. - tracer.add_traced( - Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) + tracer.add_traced(Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) return tracer def on_after_run(self, end_script_globals): import torch + # 1. Once the script has finished to execute, # we can collect its globals and access any objects. trainer = end_script_globals["cli"].trainer @@ -135,10 +135,12 @@ def on_after_run(self, end_script_globals): class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement""" + """Left to the users to implement.""" + class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement""" + """Left to the users to implement.""" + GITHUB_REPO_RUNNERS = { "PyTorch Lightning": PyTorchLightningGithubRepoRunner, @@ -199,4 +201,4 @@ def configure_layout(self): return selection_tab + run_tabs -app = LightningApp(RootFlow()) \ No newline at end of file +app = LightningApp(RootFlow()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py index 330243e5cdf01..e846c6f920681 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py @@ -2,11 +2,12 @@ import datetime as dt import panel as pn +from app_state_watcher import AppStateWatcher -import lightning as L # Todo: change import from panel_frontend import PanelFrontend -from app_state_watcher import AppStateWatcher + +import lightning as L pn.extension(sizing_mode="stretch_width") diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py index 39391c9300f30..1677cb0ce7bff 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py @@ -1,12 +1,15 @@ # app.py import panel as pn +from app_state_watcher import AppStateWatcher -import lightning as L # Todo: Change import from panel_frontend import PanelFrontend -from app_state_watcher import AppStateWatcher + +import lightning as L + pn.extension(sizing_mode="stretch_width") + def your_panel_app(app: AppStateWatcher): submit_button = pn.widgets.Button(name="submit") diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py index 82e57673e48ad..e4439a3e724d6 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py @@ -1,5 +1,4 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the -app state changes.""" +"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" # Todo: Refactor with Streamlit # Note: It would be nice one day to just watch changes within the Flow scope instead of whole app from __future__ import annotations diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py index 7fc9f493065c9..7a43f2601e810 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py @@ -11,9 +11,9 @@ import os import param - from app_state_comm import watch_app_state from other import get_flow_state + from lightning_app.utilities.imports import requires from lightning_app.utilities.state import AppState diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py index 76ceddf343b29..8bbf7f463133d 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py @@ -26,8 +26,8 @@ def __init__( cloud_compute: Optional[CloudCompute] = None, **kwargs, ): - """The GithubRepoRunner Component clones a repo, - runs a specific script with provided arguments and collect logs. + """The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect + logs. Arguments: id: Identified of the component. @@ -43,7 +43,7 @@ def __init__( cloud_compute=cloud_compute, cloud_build_config=BuildConfig(requirements=requirements), ) - self.script_path=script_path + self.script_path = script_path self.id = id self.github_repo = github_repo self.kwargs = kwargs @@ -57,8 +57,7 @@ def run(self, *args, **kwargs): # 2: Use git command line to clone the repo. repo_name = self.github_repo.split("/")[-1].replace(".git", "") cwd = os.path.dirname(__file__) - subprocess.Popen( - f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() + subprocess.Popen(f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() # 3: Execute the parent run method of the TracerPythonScript class. os.chdir(os.path.join(cwd, repo_name)) @@ -74,7 +73,6 @@ def configure_layout(self): class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.best_model_path = None @@ -106,12 +104,12 @@ def trainer_pre_fn(self, *args, work=None, **kwargs): # 5. Patch the `__init__` method of the Trainer # to inject our callback with a reference to the work. - tracer.add_traced( - Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) + tracer.add_traced(Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) return tracer def on_after_run(self, end_script_globals): import torch + # 1. Once the script has finished to execute, # we can collect its globals and access any objects. trainer = end_script_globals["cli"].trainer @@ -137,10 +135,12 @@ def on_after_run(self, end_script_globals): class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement""" + """Left to the users to implement.""" + class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement""" + """Left to the users to implement.""" + GITHUB_REPO_RUNNERS = { "PyTorch Lightning": PyTorchLightningGithubRepoRunner, @@ -189,8 +189,9 @@ def configure_layout(self): def render_fn(state: AppState): import json + with open("state.json", "w") as fp: - json.dump(state._state,fp) + json.dump(state._state, fp) import streamlit as st def page_create_new_run(): @@ -275,4 +276,4 @@ def configure_layout(self): return selection_tab + run_tabs -app = LightningApp(RootFlow()) \ No newline at end of file +app = LightningApp(RootFlow()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh b/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh index 7bb011e5faf28..07bef75470fb6 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh +++ b/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh @@ -1,2 +1,2 @@ cp -r ../../../../../../src/lightning_app/frontend/panel/*.py . -cp -r ../../../../../../src/lightning_app/frontend/utilities/*.py . \ No newline at end of file +cp -r ../../../../../../src/lightning_app/frontend/utilities/*.py . diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/other.py b/docs/source-app/workflows/add_web_ui/panel/examples/other.py index d5a650e100e6a..429d19b911328 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/other.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/other.py @@ -41,7 +41,7 @@ def get_flow_state(flow: str) -> AppState: def get_allowed_hosts() -> str: - """Returns a comma separated list of host[:port] that should be allowed to connect""" + """Returns a comma separated list of host[:port] that should be allowed to connect.""" # Todo: Improve this. I don't know how to find the specific host(s). # I tried but it did not work in cloud return "*" @@ -55,9 +55,7 @@ def has_panel_autoreload() -> bool: return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] -def get_frontend_environment( - flow: str, render_fn_or_file: Callable | str, port: int, host: str -) -> os._Environ: +def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. Args: @@ -86,7 +84,6 @@ def get_frontend_environment( def is_running_locally() -> bool: """Returns True if the lightning app is running locally. - This function can be used to determine if the app is running locally and provide a better - developer experience. + This function can be used to determine if the app is running locally and provide a better developer experience. """ return "LIGHTNING_APP_STATE_URL" not in os.environ diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py index 0881712bc10c5..9607a1be09cb2 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py @@ -4,9 +4,7 @@ ACCENT = "#792EE5" pn.extension("plotly", sizing_mode="stretch_width", template="fast") -pn.state.template.param.update( - title="⚡ Hello Panel + Lightning ⚡", accent_base_color=ACCENT, header_background=ACCENT -) +pn.state.template.param.update(title="⚡ Hello Panel + Lightning ⚡", accent_base_color=ACCENT, header_background=ACCENT) pn.config.raw_css.append( """ @@ -18,7 +16,7 @@ def get_panel_theme(): - """Returns 'default' or 'dark'""" + """Returns 'default' or 'dark'.""" return pn.state.session_args.get("theme", [b"default"])[0].decode() @@ -44,7 +42,5 @@ def get_plot(length=5): length = pn.widgets.IntSlider(value=5, start=1, end=10, name="Length") -dynamic_plot = pn.panel( - pn.bind(get_plot, length=length), sizing_mode="stretch_both", config={"responsive": True} -) +dynamic_plot = pn.panel(pn.bind(get_plot, length=length), sizing_mode="stretch_both", config={"responsive": True}) pn.Column(length, dynamic_plot).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py index 7d94b13459314..b9bbcb1fc74c1 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py @@ -43,6 +43,7 @@ class PanelFrontend(Frontend): import lightning as L from lightning_app.frontend.panel import PanelFrontend + class LitPanel(L.LightningFlow): def __init__(self): super().__init__() @@ -50,6 +51,8 @@ def __init__(self): def configure_layout(self): return self._frontend + + class LitApp(L.LightningFlow): def __init__(self): super().__init__() @@ -60,7 +63,7 @@ def configure_layout(self): app = L.LightningApp(LitApp()) - + Please note the Panel server will be logging output to error.log and output.log files respectively. @@ -90,10 +93,10 @@ def __init__(self, render_fn_or_file: Callable | str): self.render_fn_or_file = render_fn_or_file self._process: None | subprocess.Popen = None - self._log_files: Dict[str] = {} + self._log_files: dict[str] = {} _logger.debug("initialized") - def _get_popen_args(self, host: str, port: int) -> List: + def _get_popen_args(self, host: str, port: int) -> list: if callable(self.render_fn_or_file): path = str(pathlib.Path(__file__).parent / "panel_serve_render_fn.py") else: @@ -136,9 +139,7 @@ def start_server(self, host: str, port: int) -> None: if not is_running_locally(): self._open_log_files() - self._process = subprocess.Popen( # pylint: disable=consider-using-with - command, env=env, **self._log_files - ) + self._process = subprocess.Popen(command, env=env, **self._log_files) # pylint: disable=consider-using-with def stop_server(self) -> None: if self._process is None: @@ -152,7 +153,7 @@ def _close_log_files(self): file_.close() self._log_files = {} - def _open_log_files(self) -> Dict: + def _open_log_files(self) -> dict: # Don't log to file when developing locally. Makes it harder to debug. self._close_log_files() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py index 32cbc4aee6782..1e601ff51a926 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py @@ -15,6 +15,7 @@ def to_str(value): return "\n".join([str(item) for item in value]) return str(value) + if "LIGHTNING_FLOW_NAME" in os.environ: app = AppStateWatcher() else: @@ -23,9 +24,8 @@ class AppMock(param.Parameterized): state = param.Parameter() app = AppMock(state=Mock()) - - with open("state.json", "r") as fp: + with open("state.json") as fp: app.state._state = json.load(fp) app.state.requests = [ { @@ -68,20 +68,24 @@ class AppMock(param.Parameterized): ) pn.extension("terminal", sizing_mode="stretch_width", template="fast", notifications=True) -pn.state.template.param.update(site="Panel Lightning ⚡", title="Github Model Runner", accent_base_color=ACCENT, header_background=ACCENT) +pn.state.template.param.update( + site="Panel Lightning ⚡", title="Github Model Runner", accent_base_color=ACCENT, header_background=ACCENT +) pn.pane.JSON.param.hover_preview.default = True -#region: Panel extensions +# region: Panel extensions + def _to_value(value): if hasattr(value, "value"): return value.value return value + def bind_as_form(function, *args, submit, watch=False, **kwargs): - """Extends pn.bind to support "Forms" like binding. I.e. triggering only when a Submit button is clicked, - but using the dynamic values of widgets or Parameters as inputs. - + """Extends pn.bind to support "Forms" like binding. I.e. triggering only when a Submit button is clicked, but + using the dynamic values of widgets or Parameters as inputs. + Args: function (_type_): The function to execute submit (_type_): The Submit widget or parameter to depend on @@ -96,27 +100,32 @@ def bind_as_form(function, *args, submit, watch=False, **kwargs): kwargs = {} def function_wrapper(_, args=args, kwargs=kwargs): - args=[_to_value[value] for value in args] - kwargs={key: _to_value(value) for key, value in kwargs.items()} + args = [_to_value[value] for value in args] + kwargs = {key: _to_value(value) for key, value in kwargs.items()} return function(*args, **kwargs) + return pn.bind(function_wrapper, submit, watch=watch) + def show_value(widget): - """Shows the value of the widget or Parameter in a Panel - + """Shows the value of the widget or Parameter in a Panel. + Dynamically updated when ever the value changes """ + def show(value): - return pn.panel(value, sizing_mode="stretch_both") - + return pn.panel(value, sizing_mode="stretch_both") + return pn.bind(show, value=widget) + THEME = pn.state.session_args.get("theme", [b"default"])[0].decode() -pn.pane.JSON.param.theme.default = THEME if THEME=="dark" else "light" +pn.pane.JSON.param.theme.default = THEME if THEME == "dark" else "light" + +# endregion: Panel extensions +# region: Create new run -#endregion: Panel extensions -#region: Create new run def create_new_run(id, github_repo, script_path, script_args, requirements, ml_framework): new_request = { @@ -130,7 +139,8 @@ def create_new_run(id, github_repo, script_path, script_args, requirements, ml_f }, } app.state.requests = app.state.requests + [new_request] - pn.state.notifications.send("New run created", background=ACCENT, icon='⚡') + pn.state.notifications.send("New run created", background=ACCENT, icon="⚡") + def message_or_button(ml_framework, submit_button): if ml_framework not in ("PyTorch Lightning"): @@ -172,7 +182,7 @@ def create_new_page(): requirements=requirements_input, ml_framework=ml_framework_input, submit=submit_button, - watch=True + watch=True, ) return pn.Column( @@ -183,36 +193,42 @@ def create_new_page(): script_args_input, requirements_input, ml_framework_input, - pn.bind(partial(message_or_button, submit_button=submit_button), ml_framework_input), + pn.bind(partial(message_or_button, submit_button=submit_button), ml_framework_input), ) -#endregion: Create new run -#region: Run list page + +# endregion: Create new run +# region: Run list page + def configuration_component(request): return pn.pane.JSON(request, depth=4) + def work_state_ex_logs_component(work): w = work["vars"].copy() if "logs" in w: w.pop("logs") return pn.pane.JSON(w, depth=4) + def log_component(work): return pn.Column( - pn.pane.Markdown( - "```bash\n" + to_str(work["vars"]["logs"]) + "\n```", max_height=500 - ), scroll=True, css_classes=["log-container"] + pn.pane.Markdown("```bash\n" + to_str(work["vars"]["logs"]) + "\n```", max_height=500), + scroll=True, + css_classes=["log-container"], ) + def run_component(idx, request, state): work = state["structures"]["ws"]["works"][f"w_{idx}"] - name=work["vars"]["id"] + name = work["vars"]["id"] return pn.Tabs( ("Configuration", configuration_component(request)), ("Work state", work_state_ex_logs_component(work)), ("Logs", log_component(work)), - name=f"Run {idx}: {name}", margin=(5,0,0,0) + name=f"Run {idx}: {name}", + margin=(5, 0, 0, 0), ) @@ -221,11 +237,12 @@ def view_run_list_page(state: AppState): title = "# View your runs 🎈" layout = pn.Tabs(sizing_mode="stretch_both") for idx, request in enumerate(state.requests): - layout.append(run_component(idx, request, state._state)) + layout.append(run_component(idx, request, state._state)) return pn.Column(title, layout) -#endregion: Run list page -#region: App state page + +# endregion: Run list page +# region: App state page @pn.depends(app.param.state) @@ -234,12 +251,13 @@ def view_app_state_page(state: AppState): json_output = pn.pane.JSON(state._state, depth=6) return pn.Column(title, pn.Column(json_output, scroll=True, css_classes=["state-container"])) -#endregion: App state page -#region: App + +# endregion: App state page +# region: App pn.Tabs( ("New Run", create_new_page), ("View Runs", view_run_list_page), ("View State", view_app_state_page), sizing_mode="stretch_both", ).servable() -#endregion: App +# endregion: App diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py index 4abe311da62f7..c607557dd8d26 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py +++ b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py @@ -17,7 +17,6 @@ import os import panel as pn - from app_state_watcher import AppStateWatcher from other import get_render_fn_from_environment diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/state.json b/docs/source-app/workflows/add_web_ui/panel/examples/state.json index cb7d2190bb95e..ab650e70a94ad 100644 --- a/docs/source-app/workflows/add_web_ui/panel/examples/state.json +++ b/docs/source-app/workflows/add_web_ui/panel/examples/state.json @@ -1 +1 @@ -{"vars": {"_paths": {}, "_layout": {"target": "http://localhost:56891/root.flow"}, "requests": [{"id": "my_first_run", "train": {"github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", "script_path": "train_script.py", "script_args": ["--trainer.max_epochs=5", "--trainer.limit_train_batches=4", "--trainer.limit_val_batches=4", "--trainer.callbacks=ModelCheckpoint", "--trainer.callbacks.monitor=val_acc"], "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], "ml_framework": "PyTorch Lightning"}, "best_model_score": 0.21875, "best_model_path": "root.flow.ws.w_0.pt"}]}, "calls": {}, "flows": {}, "works": {}, "structures": {"ws": {"works": {"w_0": {"vars": {"_url": "http://127.0.0.1:56955", "logs": "\rSanity Checking: 0it [00:00, ?it/s]\rSanity Checking: 0%| | 0/2 [00:00 List: + def _get_popen_args(self, host: str, port: int) -> list: if callable(self.render_fn_or_file): path = str(pathlib.Path(__file__).parent / "panel_serve_render_fn.py") else: @@ -137,9 +140,7 @@ def start_server(self, host: str, port: int) -> None: if not is_running_locally(): self._open_log_files() - self._process = subprocess.Popen( # pylint: disable=consider-using-with - command, env=env, **self._log_files - ) + self._process = subprocess.Popen(command, env=env, **self._log_files) # pylint: disable=consider-using-with def stop_server(self) -> None: if self._process is None: @@ -153,7 +154,7 @@ def _close_log_files(self): file_.close() self._log_files = {} - def _open_log_files(self) -> Dict: + def _open_log_files(self) -> dict: # Don't log to file when developing locally. Makes it harder to debug. self._close_log_files() diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 82e57673e48ad..e4439a3e724d6 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -1,5 +1,4 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the -app state changes.""" +"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" # Todo: Refactor with Streamlit # Note: It would be nice one day to just watch changes within the Flow scope instead of whole app from __future__ import annotations diff --git a/src/lightning_app/frontend/utilities/other.py b/src/lightning_app/frontend/utilities/other.py index d5a650e100e6a..429d19b911328 100644 --- a/src/lightning_app/frontend/utilities/other.py +++ b/src/lightning_app/frontend/utilities/other.py @@ -41,7 +41,7 @@ def get_flow_state(flow: str) -> AppState: def get_allowed_hosts() -> str: - """Returns a comma separated list of host[:port] that should be allowed to connect""" + """Returns a comma separated list of host[:port] that should be allowed to connect.""" # Todo: Improve this. I don't know how to find the specific host(s). # I tried but it did not work in cloud return "*" @@ -55,9 +55,7 @@ def has_panel_autoreload() -> bool: return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] -def get_frontend_environment( - flow: str, render_fn_or_file: Callable | str, port: int, host: str -) -> os._Environ: +def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. Args: @@ -86,7 +84,6 @@ def get_frontend_environment( def is_running_locally() -> bool: """Returns True if the lightning app is running locally. - This function can be used to determine if the app is running locally and provide a better - developer experience. + This function can be used to determine if the app is running locally and provide a better developer experience. """ return "LIGHTNING_APP_STATE_URL" not in os.environ diff --git a/tests/tests_app/frontend/conftest.py b/tests/tests_app/frontend/conftest.py index 6effc5e7b828e..981eaaac16c7f 100644 --- a/tests/tests_app/frontend/conftest.py +++ b/tests/tests_app/frontend/conftest.py @@ -1,4 +1,4 @@ -"""Test configuration""" +"""Test configuration.""" # pylint: disable=protected-access import os from unittest import mock @@ -38,16 +38,19 @@ def _request_state(self): _state = APP_STATE self._store_state(_state) + @pytest.fixture() def flow(): return FLOW + @pytest.fixture(autouse=True, scope="module") def mock_request_state(): """Avoid requests to the api.""" with mock.patch("lightning_app.utilities.state.AppState._request_state", _request_state): yield + def do_nothing(): """Be lazy!""" @@ -61,13 +64,13 @@ def mock_start_websocket(): @pytest.fixture def app_state_state(): - """Returns an AppState dict""" + """Returns an AppState dict.""" return APP_STATE.copy() @pytest.fixture def flow_state_state(): - """Returns an AppState dict scoped to the flow""" + """Returns an AppState dict scoped to the flow.""" return FLOW_STATE.copy() @@ -84,4 +87,4 @@ def flow_state_state(): # "LIGHTNING_RENDER_PORT": f"{PORT}", # }, # ): -# yield \ No newline at end of file +# yield diff --git a/tests/tests_app/frontend/panel/app_panel.py b/tests/tests_app/frontend/panel/app_panel.py index a0ef75f7f31d9..92822c72d176d 100644 --- a/tests/tests_app/frontend/panel/app_panel.py +++ b/tests/tests_app/frontend/panel/app_panel.py @@ -1,4 +1,4 @@ -"""Test App""" +"""Test App.""" import panel as pn pn.pane.Markdown("# Panel App").servable() diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index 5b38627386c65..258fb57a8ca38 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -25,21 +25,20 @@ def _noop_render_fn(_): class MockFlow(LightningFlow): - """Test Flow""" + """Test Flow.""" @property def name(self): - """Return name""" + """Return name.""" return "root.my.flow" def run(self): # pylint: disable=arguments-differ - "Be lazy!" + """Be lazy!""" @mock.patch("lightning_app.frontend.panel.panel_frontend.subprocess") def test_panel_frontend_start_stop_server(subprocess_mock): - """Test that `PanelFrontend.start_server()` invokes subprocess.Popen with the right - parameters.""" + """Test that `PanelFrontend.start_server()` invokes subprocess.Popen with the right parameters.""" # Given frontend = PanelFrontend(render_fn_or_file=_noop_render_fn) frontend.flow = MockFlow() @@ -98,14 +97,13 @@ def _call_me(state): }, ) def test_panel_wrapper_calls_render_fn_or_file(*_): - """Run the panel_serve_render_fn_or_file""" + """Run the panel_serve_render_fn_or_file.""" runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn") # TODO: find a way to assert that _call_me got called def test_method_exception(): - """The PanelFrontend does not support render_fn_or_file being a method - and should raise an Exception""" + """The PanelFrontend does not support render_fn_or_file being a method and should raise an Exception.""" class _DummyClass: def _render_fn(self): @@ -116,7 +114,7 @@ def _render_fn(self): def test_open_close_log_files() -> bool: - """We can open and close the log files""" + """We can open and close the log files.""" frontend = PanelFrontend(_noop_render_fn) assert not frontend._log_files # When diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 3852613f8cd96..ef9ca67881675 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -1,5 +1,4 @@ -"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with -Lightning. +"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with Lightning. These tests are for serving a render_fn function. """ @@ -47,7 +46,7 @@ def test_get_view_fn_args(): def render_fn_no_args(): - """Test function with no arguments""" + """Test function with no arguments.""" return "no_args" diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index 35d0368c3ffb3..ad7a7113c43d9 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -1,14 +1,9 @@ -"""The watch_app_state function enables us to trigger a callback function when -ever the app state changes.""" +"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" import os from unittest import mock from lightning_app.core.constants import APP_SERVER_PORT -from lightning_app.frontend.utilities.app_state_comm import ( - _get_ws_url, - _run_callbacks, - watch_app_state, -) +from lightning_app.frontend.utilities.app_state_comm import _get_ws_url, _run_callbacks, watch_app_state FLOW_SUB = "lit_flow" FLOW = f"root.{FLOW_SUB}" diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index cd1e64257f02f..60ecbd1ed63d3 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,4 +1,4 @@ -"""The AppStateWatcher enables a Frontend to +"""The AppStateWatcher enables a Frontend to. - subscribe to app state changes. - to access and change the app state. @@ -65,8 +65,8 @@ def test_update_flow_state(flow_state_state: dict): def test_is_singleton(): """The AppStateWatcher is a singleton for efficiency reasons. - Its key that __new__ and __init__ of AppStateWatcher is only called once. - See https://github.com/holoviz/param/issues/643 + Its key that __new__ and __init__ of AppStateWatcher is only called once. See + https://github.com/holoviz/param/issues/643 """ # When app1 = AppStateWatcher() diff --git a/tests/tests_app/frontend/utilities/test_other.py b/tests/tests_app/frontend/utilities/test_other.py index eb278d80a6a73..dc531efbd79c7 100644 --- a/tests/tests_app/frontend/utilities/test_other.py +++ b/tests/tests_app/frontend/utilities/test_other.py @@ -25,7 +25,7 @@ def test_get_flow_state(flow_state_state: dict, flow): def render_fn(): - """Do nothing""" + """Do nothing.""" def test_get_render_fn_from_environment(): @@ -44,9 +44,7 @@ def some_fn(_): def test_get_frontend_environment_fn(): """We have a utility function to get the frontend render_fn environment.""" # When - env = get_frontend_environment( - flow="root.lit_frontend", render_fn_or_file=some_fn, host="myhost", port=1234 - ) + env = get_frontend_environment(flow="root.lit_frontend", render_fn_or_file=some_fn, host="myhost", port=1234) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" @@ -58,9 +56,7 @@ def test_get_frontend_environment_fn(): def test_get_frontend_environment_file(): """We have a utility function to get the frontend render_fn environment.""" # When - env = get_frontend_environment( - flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234 - ) + env = get_frontend_environment(flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" @@ -90,18 +86,18 @@ def test_get_frontend_environment_file(): ), ) def test_has_panel_autoreload(value, expected): - """We can get and set autoreload via the environment variable PANEL_AUTORELOAD""" + """We can get and set autoreload via the environment variable PANEL_AUTORELOAD.""" with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): assert has_panel_autoreload() == expected @mock.patch.dict(os.environ, clear=True) def test_is_running_locally() -> bool: - """We can determine if lightning is running locally""" + """We can determine if lightning is running locally.""" assert is_running_locally() @mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "127.0.0.1"}) def test_is_running_cloud() -> bool: - """We can determine if lightning is running in cloud""" + """We can determine if lightning is running in cloud.""" assert not is_running_locally() From b4cc495628d963247665341697d16304bda780de Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 29 Jul 2022 17:20:01 +0200 Subject: [PATCH 035/103] clean up PR --- .../add_web_ui/panel/examples/__init__.py | 0 .../add_web_ui/panel/examples/app_basic.py | 25 -- .../panel/examples/app_basic_script.py | 29 -- .../panel/examples/app_github_runner.py | 204 ------------- .../examples/app_interact_from_component.py | 55 ---- .../examples/app_interact_from_frontend.py | 50 ---- .../panel/examples/app_state_comm.py | 88 ------ .../panel/examples/app_state_watcher.py | 101 ------- .../panel/examples/app_streamlit.py | 31 -- .../examples/app_streamlit_github_render.py | 279 ------------------ .../add_web_ui/panel/examples/copy.sh | 2 - .../add_web_ui/panel/examples/other.py | 89 ------ .../panel/examples/panel_app_basic.py | 46 --- .../panel/examples/panel_frontend.py | 164 ---------- .../panel/examples/panel_github_runner.py | 263 ----------------- .../add_web_ui/panel/examples/panel_script.py | 13 - .../panel/examples/panel_serve_render_fn.py | 43 --- .../add_web_ui/panel/examples/state.json | 1 - .../add_web_ui/panel/tips_and_tricks.rst | 9 - 19 files changed, 1492 deletions(-) delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/__init__.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/copy.sh delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/other.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py delete mode 100644 docs/source-app/workflows/add_web_ui/panel/examples/state.json delete mode 100644 docs/source-app/workflows/add_web_ui/panel/tips_and_tricks.rst diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/__init__.py b/docs/source-app/workflows/add_web_ui/panel/examples/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py deleted file mode 100644 index 9f41f095d66a9..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic.py +++ /dev/null @@ -1,25 +0,0 @@ -# from lightning_app.frontend.panel import PanelFrontend -from panel_frontend import PanelFrontend - -import lightning as L - - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend("panel_app_basic.py") - - def configure_layout(self): - return self._frontend - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py deleted file mode 100644 index 3d427a1ce6062..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_basic_script.py +++ /dev/null @@ -1,29 +0,0 @@ -# app.py -import panel as pn - -# Todo: change import -# from lightning_app.frontend.panel import PanelFrontend -from panel_frontend import PanelFrontend - -import lightning as L - - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend("panel_script.py") - - def configure_layout(self): - return self._frontend - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py deleted file mode 100644 index e5bfc87f32104..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_github_runner.py +++ /dev/null @@ -1,204 +0,0 @@ -import io -import os -import subprocess -import sys -from copy import deepcopy -from functools import partial -from subprocess import Popen -from typing import Dict, List, Optional - -from panel_frontend import PanelFrontend - -from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow -from lightning.app import structures -from lightning.app.components.python import TracerPythonScript -from lightning.app.storage.path import Path - - -class GithubRepoRunner(TracerPythonScript): - def __init__( - self, - id: str, - github_repo: str, - script_path: str, - script_args: List[str], - requirements: List[str], - cloud_compute: Optional[CloudCompute] = None, - **kwargs, - ): - """The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect - logs. - - Arguments: - id: Identified of the component. - github_repo: The Github Repo URL to clone. - script_path: The path to the script to execute. - script_args: The arguments to be provided to the script. - requirements: The python requirements tp run the script. - cloud_compute: The object to select the cloud instance. - """ - super().__init__( - script_path=__file__, - script_args=script_args, - cloud_compute=cloud_compute, - cloud_build_config=BuildConfig(requirements=requirements), - ) - self.script_path = script_path - self.id = id - self.github_repo = github_repo - self.kwargs = kwargs - self.logs = [] - - def run(self, *args, **kwargs): - # 1. Hack: Patch stdout so we can capture the logs. - string_io = io.StringIO() - sys.stdout = string_io - - # 2: Use git command line to clone the repo. - repo_name = self.github_repo.split("/")[-1].replace(".git", "") - cwd = os.path.dirname(__file__) - subprocess.Popen(f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() - - # 3: Execute the parent run method of the TracerPythonScript class. - os.chdir(os.path.join(cwd, repo_name)) - super().run(*args, **kwargs) - - # 4: Get all the collected logs and add them to the state. - # This isn't optimal as heavy, but works for this demo purpose. - self.logs = string_io.getvalue() - string_io.close() - - def configure_layout(self): - return {"name": self.id, "content": self} - - -class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.best_model_path = None - self.best_model_score = None - - def configure_tracer(self): - from pytorch_lightning import Trainer - from pytorch_lightning.callbacks import Callback - - tracer = super().configure_tracer() - - class TensorboardServerLauncher(Callback): - def __init__(self, work): - # The provided `work` is the - # current ``PyTorchLightningScript`` work. - self.w = work - - def on_train_start(self, trainer, *_): - # Add `host` and `port` for tensorboard to work in the cloud. - cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" - server_args = f"--host {self.w.host} --port {self.w.port}" - Popen(cmd + " " + server_args, shell=True) - - def trainer_pre_fn(self, *args, work=None, **kwargs): - # Intercept Trainer __init__ call - # and inject a ``TensorboardServerLauncher`` component. - kwargs["callbacks"].append(TensorboardServerLauncher(work)) - return {}, args, kwargs - - # 5. Patch the `__init__` method of the Trainer - # to inject our callback with a reference to the work. - tracer.add_traced(Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) - return tracer - - def on_after_run(self, end_script_globals): - import torch - - # 1. Once the script has finished to execute, - # we can collect its globals and access any objects. - trainer = end_script_globals["cli"].trainer - checkpoint_callback = trainer.checkpoint_callback - lightning_module = trainer.lightning_module - - # 2. From the checkpoint_callback, - # we are accessing the best model weights - checkpoint = torch.load(checkpoint_callback.best_model_path) - - # 3. Load the best weights and torchscript the model. - lightning_module.load_state_dict(checkpoint["state_dict"]) - lightning_module.to_torchscript(f"{self.name}.pt") - - # 4. Use lightning.app.storage.Pathto create a reference to the - # torch scripted model. In the cloud with multiple machines, - # by simply passing this reference to another work, - # it triggers automatically a file transfer. - self.best_model_path = Path(f"{self.name}.pt") - - # 5. Keep track of the metrics. - self.best_model_score = float(checkpoint_callback.best_model_score) - - -class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -GITHUB_REPO_RUNNERS = { - "PyTorch Lightning": PyTorchLightningGithubRepoRunner, - "Keras": KerasGithubRepoRunner, - "Tensorflow": TensorflowGithubRepoRunner, -} - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Keep track of the requests within the state - self.requests = [] - # 2: Create a dictionary of components. - self.ws = structures.Dict() - - def run(self): - # Iterate continuously over all requests - for request_id, request in enumerate(self.requests): - self._handle_request(request_id, deepcopy(request)) - - def _handle_request(self, request_id: int, request: Dict): - # 1: Create a name and find selected framework - name = f"w_{request_id}" - ml_framework = request["train"].pop("ml_framework") - - # 2: If the component hasn't been created yet, create it. - if name not in self.ws: - work_cls = GITHUB_REPO_RUNNERS[ml_framework] - work = work_cls(id=request["id"], **request["train"]) - self.ws[name] = work - - # 3: Run the component - self.ws[name].run() - - # 4: Once the component has finished, add metadata to the request. - if self.ws[name].best_model_path: - request = self.requests[request_id] - request["best_model_score"] = self.ws[name].best_model_score - request["best_model_path"] = self.ws[name].best_model_path - - def configure_layout(self): - # Create a StreamLit UI for the user to run his Github Repo. - return PanelFrontend("panel_github_runner.py") - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.flow = Flow() - - def run(self): - self.flow.run() - - def configure_layout(self): - selection_tab = [{"name": "Run your Github Repo", "content": self.flow}] - run_tabs = [e.configure_layout() for e in self.flow.ws.values()] - return selection_tab + run_tabs - - -app = LightningApp(RootFlow()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py deleted file mode 100644 index e846c6f920681..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_component.py +++ /dev/null @@ -1,55 +0,0 @@ -# app.py -import datetime as dt - -import panel as pn -from app_state_watcher import AppStateWatcher - -# Todo: change import -from panel_frontend import PanelFrontend - -import lightning as L - -pn.extension(sizing_mode="stretch_width") - - -def your_panel_app(app: AppStateWatcher): - @pn.depends(app.param.state) - def last_update(_): - return f"last_update: {app.state.last_update}" - - return pn.Column( - last_update, - ) - - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - - self._frontend = PanelFrontend(your_panel_app) - self._last_update = dt.datetime.now() - self.last_update = self._last_update.isoformat() - - def run(self): - now = dt.datetime.now() - if (now - self._last_update).microseconds > 200: - self._last_update = now - self.last_update = self._last_update.isoformat() - - def configure_layout(self): - return self._frontend - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def run(self) -> None: - self.lit_panel.run() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py deleted file mode 100644 index 1677cb0ce7bff..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_interact_from_frontend.py +++ /dev/null @@ -1,50 +0,0 @@ -# app.py -import panel as pn -from app_state_watcher import AppStateWatcher - -# Todo: Change import -from panel_frontend import PanelFrontend - -import lightning as L - -pn.extension(sizing_mode="stretch_width") - - -def your_panel_app(app: AppStateWatcher): - - submit_button = pn.widgets.Button(name="submit") - - @pn.depends(submit_button, watch=True) - def submit(_): - app.state.count += 1 - - @pn.depends(app.param.state) - def current_count(_): - return f"current count: {app.state.count}" - - return pn.Column( - submit_button, - current_count, - ) - - -class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend(your_panel_app) - self.count = 0 - - def configure_layout(self): - return self._frontend - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py deleted file mode 100644 index e4439a3e724d6..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_comm.py +++ /dev/null @@ -1,88 +0,0 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" -# Todo: Refactor with Streamlit -# Note: It would be nice one day to just watch changes within the Flow scope instead of whole app -from __future__ import annotations - -import asyncio -import logging -import os -import threading -from typing import Callable - -import websockets - -from lightning_app.core.constants import APP_SERVER_PORT - -_logger = logging.getLogger(__name__) - -_CALLBACKS = [] -_THREAD: None | threading.Thread = None - - -def _get_ws_port(): - if "LIGHTNING_APP_STATE_URL" in os.environ: - return 8080 - return APP_SERVER_PORT - - -def _get_ws_url(): - port = _get_ws_port() - return f"ws://localhost:{port}/api/v1/ws" - - -def _run_callbacks(): - for callback in _CALLBACKS: - callback() - - -def _target_fn(): - async def update_fn(): - ws_url = _get_ws_url() - _logger.debug("connecting to web socket %s", ws_url) - async with websockets.connect(ws_url) as websocket: # pylint: disable=no-member - while True: - await websocket.recv() - # Note: I have not seen use cases where the two lines below are needed - # Changing '< 0.2' to '< 1' makes the app very sluggish to the end user - # Also the implementation can make the app state get behind because only 1 update - # is received per 0.2 second (or 1 second). - # while (time.time() - last_updated) < 0.2: - # time.sleep(0.05) - - # Todo: Add some kind of throttling. If 10 messages are received within 100ms then - # there is no need to trigger the app state changed, request state and update - # 10 times. - _logger.debug("App State Changed. Running callbacks") - _run_callbacks() - - asyncio.run(update_fn()) - - -def _start_websocket(): - global _THREAD # pylint: disable=global-statement - if not _THREAD: - _logger.debug("starting thread") - _THREAD = threading.Thread(target=_target_fn) - _THREAD.setDaemon(True) - _THREAD.start() - _logger.debug("thread started") - - -def watch_app_state(callback: Callable): - """Start the process that serves the UI at the given hostname and port number. - - Arguments: - callback: A function to run when the app state changes. Must be thread safe. - - Example: - - .. code-block:: python - - def handle_state_change(): - print("The App State Changed") - watch_app_state(handle_state_change) - """ - - _CALLBACKS.append(callback) - - _start_websocket() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py deleted file mode 100644 index 7a43f2601e810..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_state_watcher.py +++ /dev/null @@ -1,101 +0,0 @@ -"""The AppStateWatcher enables a Frontend to. - -- subscribe to app state changes -- to access and change the app state. - -This is particularly useful for the PanelFrontend but can be used by other Frontends too. -""" -from __future__ import annotations - -import logging -import os - -import param -from app_state_comm import watch_app_state -from other import get_flow_state - -from lightning_app.utilities.imports import requires -from lightning_app.utilities.state import AppState - -_logger = logging.getLogger(__name__) - - -class AppStateWatcher(param.Parameterized): - """The AppStateWatcher enables a Frontend to. - - - subscribe to app state changes - - to access and change the app state. - - This is particularly useful for the PanelFrontend, but can be used by - other Frontends too. - - Example: - - .. code-block:: python - - import param - - app = AppStateWatcher() - - app.state.counter = 1 - - - @param.depends(app.param.state, watch=True) - def update(state): - print(f"The counter was updated to {state.counter}") - - - app.state.counter += 1 - - This would print 'The counter was updated to 2'. - - The AppStateWatcher is build on top of Param which is a framework like dataclass, attrs and - Pydantic which additionally provides powerful and unique features for building reactive apps. - - Please note the AppStateWatcher is a singleton, i.e. only one instance is instantiated - """ - - state: AppState = param.ClassSelector( - class_=AppState, - constant=True, - doc=""" - The AppState holds the state of the app reduced to the scope of the Flow""", - ) - - def __new__(cls): - # This makes the AppStateWatcher a *singleton*. - # The AppStateWatcher is a singleton to minimize the number of requests etc.. - if not hasattr(cls, "_instance"): - cls._instance = super().__new__(cls) - return cls._instance - - @requires("param") - def __init__(self): - # Its critical to initialize only once - # See https://github.com/holoviz/param/issues/643 - if not hasattr(self, "_initilized"): - super().__init__(name="singleton") - self._start_watching() - self.param.state.allow_None = False - self._initilized = True - - # The below was observed when using mocks during testing - if not self.state: - raise Exception(".state has not been set.") - if not self.state._state: - raise Exception(".state._state has not been set.") - - def _start_watching(self): - watch_app_state(self._update_flow_state) - self._update_flow_state() - - def _get_flow_state(self) -> AppState: - flow = os.environ["LIGHTNING_FLOW_NAME"] - return get_flow_state(flow) - - def _update_flow_state(self): - # Todo: Consider whether to only update if ._state changed - # this might be much more performent - with param.edit_constant(self): - self.state = self._get_flow_state() - _logger.debug("Request app state") diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py deleted file mode 100644 index b06fb09857258..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit.py +++ /dev/null @@ -1,31 +0,0 @@ -# app.py -import os - -import streamlit as st - -import lightning as L -from lightning.app.frontend.stream_lit import StreamlitFrontend - - -def your_streamlit_app(lightning_app_state): - st.write("hello world") - st.write(lightning_app_state) - st.write(os.environ["LIGHTNING_FLOW_NAME"]) - - -class LitStreamlit(L.LightningFlow): - def configure_layout(self): - return StreamlitFrontend(render_fn=your_streamlit_app) - - -class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_streamlit = LitStreamlit() - - def configure_layout(self): - tab1 = {"name": "home", "content": self.lit_streamlit} - return tab1 - - -app = L.LightningApp(LitApp()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py b/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py deleted file mode 100644 index 8bbf7f463133d..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/app_streamlit_github_render.py +++ /dev/null @@ -1,279 +0,0 @@ -import io -import os -import subprocess -import sys -from copy import deepcopy -from functools import partial -from subprocess import Popen -from typing import Dict, List, Optional - -from lightning import BuildConfig, CloudCompute, LightningApp, LightningFlow -from lightning.app import structures -from lightning.app.components.python import TracerPythonScript -from lightning.app.frontend import StreamlitFrontend -from lightning.app.storage.path import Path -from lightning.app.utilities.state import AppState - - -class GithubRepoRunner(TracerPythonScript): - def __init__( - self, - id: str, - github_repo: str, - script_path: str, - script_args: List[str], - requirements: List[str], - cloud_compute: Optional[CloudCompute] = None, - **kwargs, - ): - """The GithubRepoRunner Component clones a repo, runs a specific script with provided arguments and collect - logs. - - Arguments: - id: Identified of the component. - github_repo: The Github Repo URL to clone. - script_path: The path to the script to execute. - script_args: The arguments to be provided to the script. - requirements: The python requirements tp run the script. - cloud_compute: The object to select the cloud instance. - """ - super().__init__( - script_path=__file__, - script_args=script_args, - cloud_compute=cloud_compute, - cloud_build_config=BuildConfig(requirements=requirements), - ) - self.script_path = script_path - self.id = id - self.github_repo = github_repo - self.kwargs = kwargs - self.logs = [] - - def run(self, *args, **kwargs): - # 1. Hack: Patch stdout so we can capture the logs. - string_io = io.StringIO() - sys.stdout = string_io - - # 2: Use git command line to clone the repo. - repo_name = self.github_repo.split("/")[-1].replace(".git", "") - cwd = os.path.dirname(__file__) - subprocess.Popen(f"git clone {self.github_repo}", cwd=cwd, shell=True).wait() - - # 3: Execute the parent run method of the TracerPythonScript class. - os.chdir(os.path.join(cwd, repo_name)) - super().run(*args, **kwargs) - - # 4: Get all the collected logs and add them to the state. - # This isn't optimal as heavy, but works for this demo purpose. - self.logs = string_io.getvalue() - string_io.close() - - def configure_layout(self): - return {"name": self.id, "content": self} - - -class PyTorchLightningGithubRepoRunner(GithubRepoRunner): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.best_model_path = None - self.best_model_score = None - - def configure_tracer(self): - from pytorch_lightning import Trainer - from pytorch_lightning.callbacks import Callback - - tracer = super().configure_tracer() - - class TensorboardServerLauncher(Callback): - def __init__(self, work): - # The provided `work` is the - # current ``PyTorchLightningScript`` work. - self.w = work - - def on_train_start(self, trainer, *_): - # Add `host` and `port` for tensorboard to work in the cloud. - cmd = f"tensorboard --logdir='{trainer.logger.log_dir}'" - server_args = f"--host {self.w.host} --port {self.w.port}" - Popen(cmd + " " + server_args, shell=True) - - def trainer_pre_fn(self, *args, work=None, **kwargs): - # Intercept Trainer __init__ call - # and inject a ``TensorboardServerLauncher`` component. - kwargs["callbacks"].append(TensorboardServerLauncher(work)) - return {}, args, kwargs - - # 5. Patch the `__init__` method of the Trainer - # to inject our callback with a reference to the work. - tracer.add_traced(Trainer, "__init__", pre_fn=partial(trainer_pre_fn, work=self)) - return tracer - - def on_after_run(self, end_script_globals): - import torch - - # 1. Once the script has finished to execute, - # we can collect its globals and access any objects. - trainer = end_script_globals["cli"].trainer - checkpoint_callback = trainer.checkpoint_callback - lightning_module = trainer.lightning_module - - # 2. From the checkpoint_callback, - # we are accessing the best model weights - checkpoint = torch.load(checkpoint_callback.best_model_path) - - # 3. Load the best weights and torchscript the model. - lightning_module.load_state_dict(checkpoint["state_dict"]) - lightning_module.to_torchscript(f"{self.name}.pt") - - # 4. Use lightning.app.storage.Pathto create a reference to the - # torch scripted model. In the cloud with multiple machines, - # by simply passing this reference to another work, - # it triggers automatically a file transfer. - self.best_model_path = Path(f"{self.name}.pt") - - # 5. Keep track of the metrics. - self.best_model_score = float(checkpoint_callback.best_model_score) - - -class KerasGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -class TensorflowGithubRepoRunner(GithubRepoRunner): - """Left to the users to implement.""" - - -GITHUB_REPO_RUNNERS = { - "PyTorch Lightning": PyTorchLightningGithubRepoRunner, - "Keras": KerasGithubRepoRunner, - "Tensorflow": TensorflowGithubRepoRunner, -} - - -class Flow(LightningFlow): - def __init__(self): - super().__init__() - # 1: Keep track of the requests within the state - self.requests = [] - # 2: Create a dictionary of components. - self.ws = structures.Dict() - - def run(self): - # Iterate continuously over all requests - for request_id, request in enumerate(self.requests): - self._handle_request(request_id, deepcopy(request)) - - def _handle_request(self, request_id: int, request: Dict): - # 1: Create a name and find selected framework - name = f"w_{request_id}" - ml_framework = request["train"].pop("ml_framework") - - # 2: If the component hasn't been created yet, create it. - if name not in self.ws: - work_cls = GITHUB_REPO_RUNNERS[ml_framework] - work = work_cls(id=request["id"], **request["train"]) - self.ws[name] = work - - # 3: Run the component - self.ws[name].run() - - # 4: Once the component has finished, add metadata to the request. - if self.ws[name].best_model_path: - request = self.requests[request_id] - request["best_model_score"] = self.ws[name].best_model_score - request["best_model_path"] = self.ws[name].best_model_path - - def configure_layout(self): - # Create a StreamLit UI for the user to run his Github Repo. - return StreamlitFrontend(render_fn=render_fn) - - -def render_fn(state: AppState): - import json - - with open("state.json", "w") as fp: - json.dump(state._state, fp) - import streamlit as st - - def page_create_new_run(): - st.markdown("# Create a new Run 🎈") - id = st.text_input("Name your run", value="my_first_run") - github_repo = st.text_input( - "Enter a Github Repo URL", value="https://github.com/Lightning-AI/lightning-quick-start.git" - ) - - default_script_args = "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 --trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" - default_requirements = "torchvision, pytorch_lightning, jsonargparse[signatures]" - - script_path = st.text_input("Enter your script to run", value="train_script.py") - script_args = st.text_input("Enter your base script arguments", value=default_script_args) - requirements = st.text_input("Enter your requirements", value=default_requirements) - ml_framework = st.radio( - "Select your ML Training Frameworks", options=["PyTorch Lightning", "Keras", "Tensorflow"] - ) - - if ml_framework not in ("PyTorch Lightning"): - st.write(f"{ml_framework} isn't supported yet.") - return - - clicked = st.button("Submit") - if clicked: - new_request = { - "id": id, - "train": { - "github_repo": github_repo, - "script_path": script_path, - "script_args": script_args.split(" "), - "requirements": requirements.split(" "), - "ml_framework": ml_framework, - }, - } - state.requests = state.requests + [new_request] - - def page_view_run_lists(): - st.markdown("# Run Lists 🎈") - for idx, request in enumerate(state.requests): - work = state._state["structures"]["ws"]["works"][f"w_{idx}"] - with st.expander(f"Expand to view Run {idx}", expanded=False): - if st.checkbox(f"Expand to view your configuration", key=str(idx)): - st.json(request) - if st.checkbox(f"Expand to view logs", key=str(idx)): - st.code(body=work["vars"]["logs"]) - if st.checkbox(f"Expand to view your work state", key=str(idx)): - work["vars"].pop("logs") - st.json(work) - best_model_score = request.get("best_model_score", None) - if best_model_score: - if st.checkbox(f"Expand to view your run performance", key=str(idx)): - st.json( - {"best_model_score": best_model_score, "best_model_path": request.get("best_model_path")} - ) - - def page_view_app_state(): - st.markdown("# App State 🎈") - st.write(state._state) - - page_names_to_funcs = { - "Create a new Run": page_create_new_run, - "View your Runs": page_view_run_lists, - "View the App state": page_view_app_state, - } - - selected_page = st.sidebar.selectbox("Select a page", page_names_to_funcs.keys()) - page_names_to_funcs[selected_page]() - - -class RootFlow(LightningFlow): - def __init__(self): - super().__init__() - self.flow = Flow() - - def run(self): - self.flow.run() - - def configure_layout(self): - selection_tab = [{"name": "Run your Github Repo", "content": self.flow}] - run_tabs = [e.configure_layout() for e in self.flow.ws.values()] - return selection_tab + run_tabs - - -app = LightningApp(RootFlow()) diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh b/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh deleted file mode 100644 index 07bef75470fb6..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/copy.sh +++ /dev/null @@ -1,2 +0,0 @@ -cp -r ../../../../../../src/lightning_app/frontend/panel/*.py . -cp -r ../../../../../../src/lightning_app/frontend/utilities/*.py . diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/other.py b/docs/source-app/workflows/add_web_ui/panel/examples/other.py deleted file mode 100644 index 429d19b911328..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/other.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Utility functions for lightning Frontends.""" -# Todo: Refactor stream_lit and streamlit_base to use this functionality - -from __future__ import annotations - -import inspect -import os -import pydoc -from typing import Callable - -from lightning_app.core.flow import LightningFlow -from lightning_app.utilities.state import AppState - - -def get_render_fn_from_environment(render_fn_name: str, render_fn_module_file: str) -> Callable: - """Returns the render_fn function to serve in the Frontend.""" - module = pydoc.importfile(render_fn_module_file) - return getattr(module, render_fn_name) - - -def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppState: - """Returns a new AppState with the scope reduced to the given flow.""" - flow_name = flow.name if isinstance(flow, LightningFlow) else flow - flow_name_parts = flow_name.split(".")[1:] # exclude root - flow_state = state - for part in flow_name_parts: - flow_state = getattr(flow_state, part) - return flow_state - - -def get_flow_state(flow: str) -> AppState: - """Returns an AppState scoped to the current Flow. - - Returns: - AppState: An AppState scoped to the current Flow. - """ - app_state = AppState() - app_state._request_state() # pylint: disable=protected-access - flow_state = _reduce_to_flow_scope(app_state, flow) - return flow_state - - -def get_allowed_hosts() -> str: - """Returns a comma separated list of host[:port] that should be allowed to connect.""" - # Todo: Improve this. I don't know how to find the specific host(s). - # I tried but it did not work in cloud - return "*" - - -def has_panel_autoreload() -> bool: - """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. - - Please note the casing of value does not matter - """ - return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] - - -def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: - """Returns an _Environ with the environment variables for serving a Frontend app set. - - Args: - flow (str): The name of the flow, for example root.lit_frontend - render_fn (Callable): A function to render - port (int): The port number, for example 54321 - host (str): The host, for example 'localhost' - - Returns: - os._Environ: An environement - """ - env = os.environ.copy() - env["LIGHTNING_FLOW_NAME"] = flow - env["LIGHTNING_RENDER_PORT"] = str(port) - env["LIGHTNING_RENDER_ADDRESS"] = str(host) - - if isinstance(render_fn_or_file, str): - env["LIGHTNING_RENDER_FILE"] = render_fn_or_file - else: - env["LIGHTNING_RENDER_FUNCTION"] = render_fn_or_file.__name__ - env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn_or_file).__file__ - - return env - - -def is_running_locally() -> bool: - """Returns True if the lightning app is running locally. - - This function can be used to determine if the app is running locally and provide a better developer experience. - """ - return "LIGHTNING_APP_STATE_URL" not in os.environ diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py deleted file mode 100644 index 9607a1be09cb2..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_app_basic.py +++ /dev/null @@ -1,46 +0,0 @@ -import panel as pn -import plotly.express as px - -ACCENT = "#792EE5" - -pn.extension("plotly", sizing_mode="stretch_width", template="fast") -pn.state.template.param.update(title="⚡ Hello Panel + Lightning ⚡", accent_base_color=ACCENT, header_background=ACCENT) - -pn.config.raw_css.append( - """ - .bk-root:first-of-type { - height: calc( 100vh - 150px ) !important; - } - """ -) - - -def get_panel_theme(): - """Returns 'default' or 'dark'.""" - return pn.state.session_args.get("theme", [b"default"])[0].decode() - - -def get_plotly_template(): - if get_panel_theme() == "dark": - return "plotly_dark" - return "plotly_white" - - -def get_plot(length=5): - xseries = [index for index in range(length + 1)] - yseries = [x**2 for x in xseries] - fig = px.line( - x=xseries, - y=yseries, - template=get_plotly_template(), - color_discrete_sequence=[ACCENT], - range_x=(0, 10), - markers=True, - ) - fig.layout.autosize = True - return fig - - -length = pn.widgets.IntSlider(value=5, start=1, end=10, name="Length") -dynamic_plot = pn.panel(pn.bind(get_plot, length=length), sizing_mode="stretch_both", config={"responsive": True}) -pn.Column(length, dynamic_plot).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py deleted file mode 100644 index b9bbcb1fc74c1..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_frontend.py +++ /dev/null @@ -1,164 +0,0 @@ -"""The PanelFrontend wraps your Panel code in your LightningFlow.""" -from __future__ import annotations - -import inspect -import logging -import pathlib -import subprocess -import sys -from typing import Callable, Dict, List - -from other import get_allowed_hosts, get_frontend_environment, has_panel_autoreload, is_running_locally - -from lightning_app.frontend.frontend import Frontend -from lightning_app.utilities.imports import requires -from lightning_app.utilities.log import get_frontend_logfile - -_logger = logging.getLogger("PanelFrontend") - - -class PanelFrontend(Frontend): - """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. - - To use this frontend, you must first install the `panel` package: - - .. code-block:: bash - - pip install panel - - Example: - - `panel_app_basic.py` - - .. code-block:: python - - import panel as pn - - pn.panel("Hello **Panel ⚡** World").servable() - - `app_basic.py` - - .. code-block:: python - - import lightning as L - from lightning_app.frontend.panel import PanelFrontend - - - class LitPanel(L.LightningFlow): - def __init__(self): - super().__init__() - self._frontend = PanelFrontend("panel_app_basic.py") - - def configure_layout(self): - return self._frontend - - - class LitApp(L.LightningFlow): - def __init__(self): - super().__init__() - self.lit_panel = LitPanel() - - def configure_layout(self): - return {"name": "home", "content": self.lit_panel} - - - app = L.LightningApp(LitApp()) - - Please note the Panel server will be logging output to error.log and output.log files - respectively. - - You can start the lightning server with Panel autoreload by setting the `PANEL_AUTORELOAD` - environment variable to 'yes': `AUTORELOAD=yes lightning run app my_app.py`. - - Args: - render_fn_or_file: A pure function or the path to a .py or .ipynb file. - The function must be a pure function that contains your Panel code. - The function can optionally accept an `AppStateWatcher` argument. - - Raises: - TypeError: Raised if the render_fn_or_file is a class method - """ - - @requires("panel") - def __init__(self, render_fn_or_file: Callable | str): - # Todo: consider renaming back to render_fn or something else short. - # Its a hazzle reading and writing such a long name - super().__init__() - - if inspect.ismethod(render_fn_or_file): - raise TypeError( - "The `PanelFrontend` doesn't support `render_fn_or_file` being a method. " - "Please, use a pure function." - ) - - self.render_fn_or_file = render_fn_or_file - self._process: None | subprocess.Popen = None - self._log_files: dict[str] = {} - _logger.debug("initialized") - - def _get_popen_args(self, host: str, port: int) -> list: - if callable(self.render_fn_or_file): - path = str(pathlib.Path(__file__).parent / "panel_serve_render_fn.py") - else: - path = pathlib.Path(self.render_fn_or_file) - - abs_path = str(path) - # The app is served at http://localhost:{port}/{flow}/{render_fn_or_file} - # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and - # seems to work fine. - command = [ - sys.executable, - "-m", - "panel", - "serve", - abs_path, - "--port", - str(port), - "--address", - host, - "--prefix", - self.flow.name, - "--allow-websocket-origin", - get_allowed_hosts(), - ] - if has_panel_autoreload(): - command.append("--autoreload") - _logger.debug("%s", command) - return command - - def start_server(self, host: str, port: int) -> None: - _logger.debug("starting server %s %s", host, port) - env = get_frontend_environment( - self.flow.name, - self.render_fn_or_file, - port, - host, - ) - command = self._get_popen_args(host, port) - - if not is_running_locally(): - self._open_log_files() - - self._process = subprocess.Popen(command, env=env, **self._log_files) # pylint: disable=consider-using-with - - def stop_server(self) -> None: - if self._process is None: - raise RuntimeError("Server is not running. Call `PanelFrontend.start_server()` first.") - self._process.kill() - self._close_log_files() - - def _close_log_files(self): - for file_ in self._log_files.values(): - if not file_.closed: - file_.close() - self._log_files = {} - - def _open_log_files(self) -> dict: - # Don't log to file when developing locally. Makes it harder to debug. - self._close_log_files() - - std_err_out = get_frontend_logfile("error.log") - std_out_out = get_frontend_logfile("output.log") - stderr = std_err_out.open("wb") - stdout = std_out_out.open("wb") - self._log_files = {"stdout": stderr, "stderr": stdout} diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py deleted file mode 100644 index 1e601ff51a926..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_github_runner.py +++ /dev/null @@ -1,263 +0,0 @@ -import json -import os -from functools import partial -from unittest.mock import Mock - -import panel as pn -import param -from app_state_watcher import AppStateWatcher - -from lightning_app.utilities.state import AppState - - -def to_str(value): - if isinstance(value, list): - return "\n".join([str(item) for item in value]) - return str(value) - - -if "LIGHTNING_FLOW_NAME" in os.environ: - app = AppStateWatcher() -else: - - class AppMock(param.Parameterized): - state = param.Parameter() - - app = AppMock(state=Mock()) - - with open("state.json") as fp: - app.state._state = json.load(fp) - app.state.requests = [ - { - "id": 0, - "train": { - "github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", - "script_path": "train_script.py", - "script_args": [ - "--trainer.max_epochs=5", - "--trainer.limit_train_batches=4", - "--trainer.limit_val_batches=4", - "--trainer.callbacks=ModelCheckpoint", - "--trainer.callbacks.monitor=val_acc", - ], - "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], - "ml_framework": "PyTorch Lightning", - }, - } - ] - - -ACCENT = "#792EE5" - -pn.config.raw_css.append( - """ - .bk-root { - height: calc( 100vh - 200px ) !important; - } - .state-container { - height: calc(100vh - 300px) !important; - } - .log-container { - height: calc(100vh - 380px) !important; - } - .scrollable { - overflow-x: hidden !important; - overflow-y: scroll !important; - } - """ -) - -pn.extension("terminal", sizing_mode="stretch_width", template="fast", notifications=True) -pn.state.template.param.update( - site="Panel Lightning ⚡", title="Github Model Runner", accent_base_color=ACCENT, header_background=ACCENT -) -pn.pane.JSON.param.hover_preview.default = True - -# region: Panel extensions - - -def _to_value(value): - if hasattr(value, "value"): - return value.value - return value - - -def bind_as_form(function, *args, submit, watch=False, **kwargs): - """Extends pn.bind to support "Forms" like binding. I.e. triggering only when a Submit button is clicked, but - using the dynamic values of widgets or Parameters as inputs. - - Args: - function (_type_): The function to execute - submit (_type_): The Submit widget or parameter to depend on - watch (bool, optional): Defaults to False. - - Returns: - _type_: A Reactive Function - """ - if not args: - args = [] - if not kwargs: - kwargs = {} - - def function_wrapper(_, args=args, kwargs=kwargs): - args = [_to_value[value] for value in args] - kwargs = {key: _to_value(value) for key, value in kwargs.items()} - return function(*args, **kwargs) - - return pn.bind(function_wrapper, submit, watch=watch) - - -def show_value(widget): - """Shows the value of the widget or Parameter in a Panel. - - Dynamically updated when ever the value changes - """ - - def show(value): - return pn.panel(value, sizing_mode="stretch_both") - - return pn.bind(show, value=widget) - - -THEME = pn.state.session_args.get("theme", [b"default"])[0].decode() -pn.pane.JSON.param.theme.default = THEME if THEME == "dark" else "light" - - -# endregion: Panel extensions -# region: Create new run - - -def create_new_run(id, github_repo, script_path, script_args, requirements, ml_framework): - new_request = { - "id": id, - "train": { - "github_repo": github_repo, - "script_path": script_path, - "script_args": script_args.split(" "), - "requirements": requirements.split(" "), - "ml_framework": ml_framework, - }, - } - app.state.requests = app.state.requests + [new_request] - pn.state.notifications.send("New run created", background=ACCENT, icon="⚡") - - -def message_or_button(ml_framework, submit_button): - if ml_framework not in ("PyTorch Lightning"): - return f"💥 {ml_framework} isn't supported yet." - else: - return submit_button - - -def create_new_page(): - id_input = pn.widgets.TextInput(name="Name your run", value="my_first_run") - github_repo_input = pn.widgets.TextInput( - name="Enter a Github Repo URL", - value="https://github.com/Lightning-AI/lightning-quick-start.git", - ) - script_path_input = pn.widgets.TextInput(name="Enter your script to run", value="train_script.py") - - script_args_input = pn.widgets.TextInput( - name="Enter your base script arguments", - value=( - "--trainer.max_epochs=5 --trainer.limit_train_batches=4 --trainer.limit_val_batches=4 " - "--trainer.callbacks=ModelCheckpoint --trainer.callbacks.monitor=val_acc" - ), - ) - requirements_input = pn.widgets.TextInput( - name="Enter your requirements", value="torchvision, pytorch_lightning, jsonargparse[signatures]" - ) - ml_framework_input = pn.widgets.RadioBoxGroup( - name="Select your ML Training Frameworks", - options=["PyTorch Lightning", "Keras", "Tensorflow"], - inline=True, - ) - submit_button = pn.widgets.Button(name="⚡ SUBMIT ⚡", button_type="primary") - bind_as_form( - create_new_run, - id=id_input, - github_repo=github_repo_input, - script_path=script_path_input, - script_args=script_args_input, - requirements=requirements_input, - ml_framework=ml_framework_input, - submit=submit_button, - watch=True, - ) - - return pn.Column( - "# Create a new run 🎈", - id_input, - github_repo_input, - script_path_input, - script_args_input, - requirements_input, - ml_framework_input, - pn.bind(partial(message_or_button, submit_button=submit_button), ml_framework_input), - ) - - -# endregion: Create new run -# region: Run list page - - -def configuration_component(request): - return pn.pane.JSON(request, depth=4) - - -def work_state_ex_logs_component(work): - w = work["vars"].copy() - if "logs" in w: - w.pop("logs") - return pn.pane.JSON(w, depth=4) - - -def log_component(work): - return pn.Column( - pn.pane.Markdown("```bash\n" + to_str(work["vars"]["logs"]) + "\n```", max_height=500), - scroll=True, - css_classes=["log-container"], - ) - - -def run_component(idx, request, state): - work = state["structures"]["ws"]["works"][f"w_{idx}"] - name = work["vars"]["id"] - return pn.Tabs( - ("Configuration", configuration_component(request)), - ("Work state", work_state_ex_logs_component(work)), - ("Logs", log_component(work)), - name=f"Run {idx}: {name}", - margin=(5, 0, 0, 0), - ) - - -@pn.depends(app.param.state) -def view_run_list_page(state: AppState): - title = "# View your runs 🎈" - layout = pn.Tabs(sizing_mode="stretch_both") - for idx, request in enumerate(state.requests): - layout.append(run_component(idx, request, state._state)) - return pn.Column(title, layout) - - -# endregion: Run list page -# region: App state page - - -@pn.depends(app.param.state) -def view_app_state_page(state: AppState): - title = "# View the full state of the app 🎈" - json_output = pn.pane.JSON(state._state, depth=6) - return pn.Column(title, pn.Column(json_output, scroll=True, css_classes=["state-container"])) - - -# endregion: App state page -# region: App -pn.Tabs( - ("New Run", create_new_page), - ("View Runs", view_run_list_page), - ("View State", view_app_state_page), - sizing_mode="stretch_both", -).servable() -# endregion: App diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py deleted file mode 100644 index b27678746d38f..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_script.py +++ /dev/null @@ -1,13 +0,0 @@ -import os - -import panel as pn - -pn.extension(sizing_mode="stretch_width") - -pn.panel("# Hello Panel 4").servable() - -from app_state_watcher import AppStateWatcher - -app = AppStateWatcher() -pn.panel(os.environ.get("PANEL_AUTORELOAD", "no")).servable() -pn.pane.JSON(app.state._state, theme="light", height=300, width=500, depth=3).servable() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py b/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py deleted file mode 100644 index c607557dd8d26..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/panel_serve_render_fn.py +++ /dev/null @@ -1,43 +0,0 @@ -"""This file gets run by Python to lunch a Panel Server with Lightning. - -From here, we will call the render_fn that the user provided to the PanelFrontend. - -It requires the below environment variables to be set - -- LIGHTNING_RENDER_FUNCTION -- LIGHTNING_RENDER_MODULE_FILE - -Example: - -.. code-block:: bash - - python panel_serve_render_fn -""" -import inspect -import os - -import panel as pn -from app_state_watcher import AppStateWatcher -from other import get_render_fn_from_environment - - -def _get_render_fn(): - render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] - render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] - render_fn = get_render_fn_from_environment(render_fn_name, render_fn_module_file) - if inspect.signature(render_fn).parameters: - - def _render_fn_wrapper(): - app = AppStateWatcher() - return render_fn(app) - - return _render_fn_wrapper - return render_fn - - -if __name__.startswith("bokeh"): - # I use caching for efficiency reasons. It shaves off 10ms from having - # to get_render_fn_from_environment every time - if not "lightning_render_fn" in pn.state.cache: - pn.state.cache["lightning_render_fn"] = _get_render_fn() - pn.state.cache["lightning_render_fn"]() diff --git a/docs/source-app/workflows/add_web_ui/panel/examples/state.json b/docs/source-app/workflows/add_web_ui/panel/examples/state.json deleted file mode 100644 index ab650e70a94ad..0000000000000 --- a/docs/source-app/workflows/add_web_ui/panel/examples/state.json +++ /dev/null @@ -1 +0,0 @@ -{"vars": {"_paths": {}, "_layout": {"target": "http://localhost:56891/root.flow"}, "requests": [{"id": "my_first_run", "train": {"github_repo": "https://github.com/Lightning-AI/lightning-quick-start.git", "script_path": "train_script.py", "script_args": ["--trainer.max_epochs=5", "--trainer.limit_train_batches=4", "--trainer.limit_val_batches=4", "--trainer.callbacks=ModelCheckpoint", "--trainer.callbacks.monitor=val_acc"], "requirements": ["torchvision,", "pytorch_lightning,", "jsonargparse[signatures]"], "ml_framework": "PyTorch Lightning"}, "best_model_score": 0.21875, "best_model_path": "root.flow.ws.w_0.pt"}]}, "calls": {}, "flows": {}, "works": {}, "structures": {"ws": {"works": {"w_0": {"vars": {"_url": "http://127.0.0.1:56955", "logs": "\rSanity Checking: 0it [00:00, ?it/s]\rSanity Checking: 0%| | 0/2 [00:00 Date: Fri, 29 Jul 2022 17:29:57 +0200 Subject: [PATCH 036/103] change video links --- .gitignore | 333 +++++++++--------- .../workflows/add_web_ui/panel/basic.rst | 12 +- 2 files changed, 173 insertions(+), 172 deletions(-) diff --git a/.gitignore b/.gitignore index ea65e03fd12a4..723d430cf951d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,166 +1,167 @@ -# project -.DS_Store -run_configs/ -model_weights/ -pip-wheel-metadata/ -lightning_logs/ -.vscode/ - -# Documentations -docs/source-app/generated -docs/source-app/*/generated -docs/source-pytorch/api -docs/source-pytorch/*.md -docs/source-pytorch/generated -docs/source-pytorch/*/generated -docs/source-pytorch/notebooks -docs/source-pytorch/_static/images/course_UvA-DL -docs/source-pytorch/_static/images/lightning_examples - -# C extensions -*.so -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -timit_data/ -grid_generated* -grid_ori* - -# PyCharm -.idea/ - -# Distribution / packaging -.Python -ide_layouts/ -build/ -_build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -src/lightning/*/ - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -tests/tests_tt_dir/ -tests/save_dir -tests/tests/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -env/ -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -# pytest -.pytest_cache/ - -# data -.data/ -Datasets/ -mnist/ -MNIST/ -tests/legacy/checkpoints/ -*.gz -*ubyte - -# pl tests -ml-runs/ -mlruns/ -*.zip -*.ckpt -test-reports/ -wandb -.forked/ -*.prof -*.tar.gz -.neptune/ - -# dataset generated from bolts in examples. -cifar-10-batches-py -*.pt -# ctags -tags -.tags - -# Lightning StreamlitFrontend or PanelFrontend -.storage - -# Personal scripts -script.* +# project +.DS_Store +run_configs/ +model_weights/ +pip-wheel-metadata/ +lightning_logs/ +.vscode/ + +# Documentations +docs/source-app/generated +docs/source-app/*/generated +docs/source-pytorch/api +docs/source-pytorch/*.md +docs/source-pytorch/generated +docs/source-pytorch/*/generated +docs/source-pytorch/notebooks +docs/source-pytorch/_static/images/course_UvA-DL +docs/source-pytorch/_static/images/lightning_examples + +# C extensions +*.so +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +timit_data/ +grid_generated* +grid_ori* + +# PyCharm +.idea/ + +# Distribution / packaging +.Python +ide_layouts/ +build/ +_build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +src/lightning/*/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +tests/tests_tt_dir/ +tests/save_dir +tests/tests/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +env/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +# pytest +.pytest_cache/ + +# data +.data/ +Datasets/ +mnist/ +MNIST/ +tests/legacy/checkpoints/ +*.gz +*ubyte + +# pl tests +ml-runs/ +mlruns/ +*.zip +*.ckpt +test-reports/ +wandb +.forked/ +*.prof +*.tar.gz +.neptune/ + +# dataset generated from bolts in examples. +cifar-10-batches-py +*.pt +# ctags +tags +.tags + +# Lightning StreamlitFrontend or PanelFrontend +.storage + +# Personal scripts +script.* +docs/source-app/workflows/add_web_ui/panel/examples/ diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index ea3d312fa6879..a549758940689 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -21,15 +21,15 @@ via `HoloViews`_, streaming and much more. - Panel is used by for example Rapids to power `CuxFilter`_, a CuDF based big data viz framework. - Panel can be deployed on your favorite server or cloud including `Lightning`_. -.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-intro.gif +.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-intro.gif :alt: Example Panel App Example Panel App Panel is **particularly well suited for lightning.ai apps** that needs to display live progress as the Panel server can react -to progress and asynchronously push messages from the server to the client via web socket communication. +to state changes and asynchronously push messages from the server to the client via web socket communication. -.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-streaming.gif +.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-streaming.gif?raw=true :alt: Example Panel Streaming App Example Panel Streaming App @@ -110,7 +110,7 @@ Run the app locally to see it! The app should look like the below -.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/images/panel-lightning/panel-lightning-basic.png +.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/images/panel-lightning/panel-lightning-basic.png :alt: Basic Panel Lightning App Basic Panel Lightning App @@ -229,7 +229,7 @@ Try running the below PANEL_AUTORELOAD=yes lightning run app app.py -.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-autoreload.gif +.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-lightning-autoreload.gif :alt: Basic Panel Lightning App with autoreload Basic Panel Lightning App with autoreload @@ -310,7 +310,7 @@ Finally run the app lightning run app app.py -.. figure:: https://cdn.jsdelivr.net/gh/MarcSkovMadsen/awesome-panel-assets@master/videos/panel-lightning/panel-lightning-theme.gif +.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-lightning-theme.gif :alt: Basic Panel Plotly Lightning App with theming Basic Panel Plotly Lightning App with theming From dd19fc2fe5561baccfac8585096623d35e1c0ab4 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 29 Jul 2022 17:34:16 +0200 Subject: [PATCH 037/103] update links --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index a549758940689..eae94337d1ac2 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -21,7 +21,7 @@ via `HoloViews`_, streaming and much more. - Panel is used by for example Rapids to power `CuxFilter`_, a CuDF based big data viz framework. - Panel can be deployed on your favorite server or cloud including `Lightning`_. -.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-intro.gif +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-intro.gif :alt: Example Panel App Example Panel App @@ -29,7 +29,7 @@ via `HoloViews`_, streaming and much more. Panel is **particularly well suited for lightning.ai apps** that needs to display live progress as the Panel server can react to state changes and asynchronously push messages from the server to the client via web socket communication. -.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-streaming.gif?raw=true +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-streaming.gif :alt: Example Panel Streaming App Example Panel Streaming App @@ -110,7 +110,7 @@ Run the app locally to see it! The app should look like the below -.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/images/panel-lightning/panel-lightning-basic.png +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/images/panel-lightning/panel-lightning-basic.png :alt: Basic Panel Lightning App Basic Panel Lightning App @@ -229,7 +229,7 @@ Try running the below PANEL_AUTORELOAD=yes lightning run app app.py -.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-lightning-autoreload.gif +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-autoreload.gif :alt: Basic Panel Lightning App with autoreload Basic Panel Lightning App with autoreload @@ -310,7 +310,7 @@ Finally run the app lightning run app app.py -.. figure:: https://github.com/MarcSkovMadsen/awesome-panel-assets/blob/master/videos/panel-lightning/panel-lightning-theme.gif +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-theme.gif :alt: Basic Panel Plotly Lightning App with theming Basic Panel Plotly Lightning App with theming From 9420a0c352ed0651b9e0a7559c7962a2342077a6 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 29 Jul 2022 17:38:14 +0200 Subject: [PATCH 038/103] fix link --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index eae94337d1ac2..9d47b078b099e 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -29,7 +29,7 @@ via `HoloViews`_, streaming and much more. Panel is **particularly well suited for lightning.ai apps** that needs to display live progress as the Panel server can react to state changes and asynchronously push messages from the server to the client via web socket communication. -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-streaming.gif +.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-streaming-intro.gif :alt: Example Panel Streaming App Example Panel Streaming App From 9c210ee611bff2623e5eb1525acc04b7cd422702 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 29 Jul 2022 17:46:42 +0200 Subject: [PATCH 039/103] minor improvements --- .../workflows/add_web_ui/panel/basic.rst | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 9d47b078b099e..0718c6643e3e1 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -134,7 +134,7 @@ In this section, we explain each part of this code in detail. ---- 0. Define a Panel app -^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ First, find the Panel app you want to integrate. In this example, that app looks like: @@ -144,7 +144,7 @@ First, find the Panel app you want to integrate. In this example, that app looks pn.panel("Hello **Panel ⚡** World").servable() -Refer to the `Panel documentation `_ or `awesome-panel.org `_ for more complex examples. +Refer to the `Panel documentation `_ and `awesome-panel.org `_ for more complex examples. ---- @@ -216,9 +216,12 @@ In this case, we render the ``LitPanel`` UI in the ``home`` tab of the applicati def configure_layout(self): return {"name": "home", "content": self.lit_panel} -********** -Autoreload -********** +************* +Tips & Tricks +************* + +0. Use autoreload while developing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To speed up your development workflow, you can run your lightning app with Panel **autoreload** by setting the environment variable ``PANEL_AUTORELOAD`` to ``yes``. @@ -234,9 +237,8 @@ Try running the below Basic Panel Lightning App with autoreload -******* -Theming -******* +1. Theme your app +^^^^^^^^^^^^^^^^^ To theme your app you, can use the lightning accent color #792EE5 with the `FastListTemplate`_. From 1d0427d7b956859f83fe23a0bf9f48eea793c779 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 29 Jul 2022 17:54:11 +0200 Subject: [PATCH 040/103] fix small issues --- .../add_web_ui/panel/intermediate.rst | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index ae8b269b849f6..4422d26c234c5 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -4,7 +4,7 @@ Add a web UI with Panel (intermediate) **Audience:** Users who want to communicate between the Lightning App and Panel. -**Prereqs:** Must have read the `panel basic `_ guide. +**Prereqs:** Must have read the `Panel basic `_ guide. ---- @@ -18,7 +18,7 @@ You can modify the state variables of a Lightning component via the ``AppStateWa For example, here we increase the ``count`` variable of the Lightning Component every time a user presses a button: -.. code:: bash +.. code:: python # app_panel.py @@ -46,7 +46,7 @@ presses a button: -.. code:: bash +.. code:: python # app.py @@ -94,12 +94,12 @@ presses a button: Interact with Panel from the component ************************************** -To update the `PanelFrontend` from any Lightning component, update the property in the component. Make sure to call ``run`` method from the +To update the `PanelFrontend` from any Lightning component, update the property in the component. Make sure to call the ``run`` method from the parent component. In this example we update the value of ``count`` from the component: -.. code:: bash +.. code:: python # app_panel.py @@ -117,7 +117,7 @@ In this example we update the value of ``count`` from the component: pn.panel(last_update).servable() -.. code:: bash +.. code:: python # app.py @@ -163,14 +163,14 @@ In this example we update the value of ``count`` from the component: Panel Lightning App updating a counter from the component -******************* -Panel Tips & Tricks -******************* +************* +Tips & Tricks +************* -- Caching: Panel provides the easy to use ```pn.state.cache` memory based, ``dict`` caching. If you are looking for something persistent try `DiskCache `_ its really powerful and simple to use. You can use it to communicate large amounts of data between the components and frontend(s). -- Notifactions: Panel provides easy to use `notifications `_. You can for example use them to provide notifications about runs starting or ending. +- Caching: Panel provides the easy to use ``pn.state.cache`` memory based, ``dict`` caching. If you are looking for something persistent try `DiskCache `_ its really powerful and simple to use. You can use it to communicate large amounts of data between the components and frontend(s). +- Notifications: Panel provides easy to use `notifications `_. You can for example use them to provide notifications about runs starting or ending. - Tabulator Table: Panel provides the `Tabulator table `_ which features expandable rows. The table is useful to provide for example an overview of you runs. But you can dig into the details by clicking and expanding the row. -- Task Scheduling: Panel provides easy to use `task scheduling `. You can use this to for example read and display files created by your components on a schedule basis. +- Task Scheduling: Panel provides easy to use `task scheduling `_. You can use this to for example read and display files created by your components on a scheduled basis. - Terminal: Panel provides the `Xterm.js terminal `_ which can be used to display live logs from your components and allow you to provide a terminal interface to your component. .. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-github-runner.gif @@ -178,4 +178,4 @@ Panel Tips & Tricks Panel Lightning App running models on github -# Todo: Add link to the code and running app. +# Todo: Add link to the code and running app. Where can I put this? From 21c5e1a2ef6247687f920d2ab84887147ff1df96 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 29 Jul 2022 18:01:05 +0200 Subject: [PATCH 041/103] fix minor spelling errors --- tests/tests_app/frontend/utilities/test_app_state_comm.py | 2 +- tests/tests_app/frontend/utilities/test_app_state_watcher.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index ad7a7113c43d9..02667b3bd8ba8 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -26,7 +26,7 @@ def test_get_ws_url_when_cloud(): @mock.patch.dict(os.environ, {"LIGHTNING_FLOW_NAME": "FLOW"}) def test_watch_app_state(): - """We can watch the app state and run a callback function when it changes.""" + """We can watch the app state and a callback function will be run when it changes.""" callback = mock.MagicMock() # When watch_app_state(callback) diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index 60ecbd1ed63d3..8f083a558364b 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,9 +1,9 @@ -"""The AppStateWatcher enables a Frontend to. +"""The AppStateWatcher enables a Frontend to - subscribe to app state changes. - to access and change the app state. -This is particularly useful for the PanelFrontend but can be used by other Frontends too. +This is particularly useful for the PanelFrontend, but can be used by other Frontends too. """ # pylint: disable=protected-access import os From e907620ab7456c566a774dbcb62725d7e95f1227 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Mon, 1 Aug 2022 08:56:09 +0200 Subject: [PATCH 042/103] update --- .gitignore | 333 ++++++++++++++++++++++++++--------------------------- 1 file changed, 166 insertions(+), 167 deletions(-) diff --git a/.gitignore b/.gitignore index 723d430cf951d..0f03c69600bed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,167 +1,166 @@ -# project -.DS_Store -run_configs/ -model_weights/ -pip-wheel-metadata/ -lightning_logs/ -.vscode/ - -# Documentations -docs/source-app/generated -docs/source-app/*/generated -docs/source-pytorch/api -docs/source-pytorch/*.md -docs/source-pytorch/generated -docs/source-pytorch/*/generated -docs/source-pytorch/notebooks -docs/source-pytorch/_static/images/course_UvA-DL -docs/source-pytorch/_static/images/lightning_examples - -# C extensions -*.so -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -timit_data/ -grid_generated* -grid_ori* - -# PyCharm -.idea/ - -# Distribution / packaging -.Python -ide_layouts/ -build/ -_build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -src/lightning/*/ - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -tests/tests_tt_dir/ -tests/save_dir -tests/tests/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -env/ -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -# pytest -.pytest_cache/ - -# data -.data/ -Datasets/ -mnist/ -MNIST/ -tests/legacy/checkpoints/ -*.gz -*ubyte - -# pl tests -ml-runs/ -mlruns/ -*.zip -*.ckpt -test-reports/ -wandb -.forked/ -*.prof -*.tar.gz -.neptune/ - -# dataset generated from bolts in examples. -cifar-10-batches-py -*.pt -# ctags -tags -.tags - -# Lightning StreamlitFrontend or PanelFrontend -.storage - -# Personal scripts -script.* -docs/source-app/workflows/add_web_ui/panel/examples/ +# project +.DS_Store +run_configs/ +model_weights/ +pip-wheel-metadata/ +lightning_logs/ +.vscode/ + +# Documentations +docs/source-app/generated +docs/source-app/*/generated +docs/source-pytorch/api +docs/source-pytorch/*.md +docs/source-pytorch/generated +docs/source-pytorch/*/generated +docs/source-pytorch/notebooks +docs/source-pytorch/_static/images/course_UvA-DL +docs/source-pytorch/_static/images/lightning_examples + +# C extensions +*.so +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +timit_data/ +grid_generated* +grid_ori* + +# PyCharm +.idea/ + +# Distribution / packaging +.Python +ide_layouts/ +build/ +_build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +src/lightning/*/ + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +tests/tests_tt_dir/ +tests/save_dir +tests/tests/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env +.env_stagging + +# virtualenv +.venv +env/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +# pytest +.pytest_cache/ + +# data +.data/ +Datasets/ +mnist/ +MNIST/ +tests/legacy/checkpoints/ +*.gz +*ubyte + +# pl tests +ml-runs/ +mlruns/ +*.zip +*.ckpt +test-reports/ +wandb +.forked/ +*.prof +*.tar.gz +.neptune/ + +# dataset generated from bolts in examples. +cifar-10-batches-py +*.pt +# ctags +tags +.tags +src/lightning_app/ui/* +*examples/template_react_ui* +hars* +artifacts/* +*docs/examples* From f88027e97382e0f82f479e16f6f83a1538030e0c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Aug 2022 08:54:36 +0000 Subject: [PATCH 043/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_app/frontend/utilities/test_app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index 8f083a558364b..aa480218fc556 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,4 +1,4 @@ -"""The AppStateWatcher enables a Frontend to +"""The AppStateWatcher enables a Frontend to. - subscribe to app state changes. - to access and change the app state. From 72b27e611c099b4791c2d8b9209a08d842cf3bda Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 11 Aug 2022 11:52:38 +0200 Subject: [PATCH 044/103] update --- requirements/app/ui.txt | 1 + tests/tests_app/utilities/test_app_logs.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements/app/ui.txt b/requirements/app/ui.txt index 09842cc07e615..7dcf56136e326 100644 --- a/requirements/app/ui.txt +++ b/requirements/app/ui.txt @@ -1,2 +1,3 @@ streamlit>=1.3.1, <=1.11.1 panel +param diff --git a/tests/tests_app/utilities/test_app_logs.py b/tests/tests_app/utilities/test_app_logs.py index e7384dd72d6e2..7a0fe087e7c29 100644 --- a/tests/tests_app/utilities/test_app_logs.py +++ b/tests/tests_app/utilities/test_app_logs.py @@ -1,4 +1,5 @@ from datetime import datetime +from time import sleep from unittest.mock import MagicMock from lightning_app.utilities.app_logs import _LogEvent @@ -6,6 +7,7 @@ def test_log_event(): event_1 = _LogEvent("", datetime.now(), MagicMock(), MagicMock()) + sleep(0.1) event_2 = _LogEvent("", datetime.now(), MagicMock(), MagicMock()) assert event_1 < event_2 assert event_1 <= event_2 From 6c39414cc4df24f6b572903672b022c0bc0b8ac4 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 11 Aug 2022 18:32:20 +0200 Subject: [PATCH 045/103] remove bad merge --- docs/source-app/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source-app/index.rst b/docs/source-app/index.rst index ed761ee911ccb..239288004c2a0 100644 --- a/docs/source-app/index.rst +++ b/docs/source-app/index.rst @@ -147,7 +147,6 @@ Keep Learning :height: 180 .. displayitem:: ->>>>>>> master :header: Hands-on Examples :description: Learn by building Apps and Components. :col_css: col-md-6 From dd1a28d60f970f63fa45377559e603dfa973e46b Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 12 Aug 2022 10:04:40 +0200 Subject: [PATCH 046/103] update --- tests/tests_app/cli/test_cmd_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_app/cli/test_cmd_install.py b/tests/tests_app/cli/test_cmd_install.py index 2d277ddb7790c..6fe69ea62dbe9 100644 --- a/tests/tests_app/cli/test_cmd_install.py +++ b/tests/tests_app/cli/test_cmd_install.py @@ -231,9 +231,9 @@ def test_proper_url_parsing(): source_url, git_url, folder_name, git_sha = cmd_install._show_install_app_prompt( component_entry, app, org, True, resource_type="app" ) - assert folder_name == "LAI-InVideo-search-App" + assert folder_name == "video_search_react" # FixMe: this need to be updated after release with updated org rename - assert source_url == "https://github.com/Lightning-AI/LAI-InVideo-search-App" + assert source_url == "https://github.com/PyTorchLightning/video_search_react" assert "#ref" not in git_url assert git_sha From 67f2e101b622a70fea5855c596e9cfadaccfdcfd Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sat, 13 Aug 2022 06:34:11 +0200 Subject: [PATCH 047/103] Update src/lightning_app/frontend/utilities/app_state_watcher.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- src/lightning_app/frontend/utilities/app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 269f98a5d1983..10a4fb3e3ba92 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -49,7 +49,7 @@ def update(state): This would print ``The counter was updated to 2``. - The AppStateWatcher is build on top of Param which is a framework like dataclass, attrs and + The AppStateWatcher is built on top of Param which is a framework like dataclass, attrs and Pydantic which additionally provides powerful and unique features for building reactive apps. Please note the AppStateWatcher is a singleton, i.e. only one instance is instantiated From 1b05d1111c68945aa5e125457f61222c35fcc039 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sat, 13 Aug 2022 04:55:05 +0000 Subject: [PATCH 048/103] simplify server_entry_point to entry_point --- .../frontend/panel/panel_frontend.py | 22 +++++++++---------- .../frontend/panel/test_panel_frontend.py | 12 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 1fd962e35063c..54cbe15a47ee5 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -68,28 +68,28 @@ def configure_layout(self): environment variable to 'yes': `AUTORELOAD=yes lightning run app app_basic.py`. Args: - server_entry_point: A pure function or the path to a .py or .ipynb file. + entry_point: A pure function or the path to a .py or .ipynb file. The function must be a pure function that contains your Panel code. The function can optionally accept an `AppStateWatcher` argument. Raises: - TypeError: Raised if the server_entry_point is a class method + TypeError: Raised if the entry_point is a class method """ @requires("panel") - def __init__(self, server_entry_point: Callable | str): + def __init__(self, entry_point: Callable | str): super().__init__() - if inspect.ismethod(server_entry_point): + if inspect.ismethod(entry_point): raise TypeError( - "The `PanelFrontend` doesn't support `server_entry_point` being a method. " + "The `PanelFrontend` doesn't support `entry_point` being a method. " "Please, use a pure function." ) - self.server_entry_point = server_entry_point + self.entry_point = entry_point self._process: None | subprocess.Popen = None self._log_files: dict[str, TextIO] = {} - _logger.debug(f"PanelFrontend Frontend with {server_entry_point} is initialized.") + _logger.debug(f"PanelFrontend Frontend with {entry_point} is initialized.") def start_server(self, host: str, port: int) -> None: _logger.debug(f"PanelFrontend starting server on {host}:{port}") @@ -97,7 +97,7 @@ def start_server(self, host: str, port: int) -> None: # 1: Prepare environment variables and arguments. env = get_frontend_environment( self.flow.name, - self.server_entry_point, + self.entry_point, port, host, ) @@ -131,13 +131,13 @@ def _open_log_files(self) -> None: self._log_files = {"stdout": stderr, "stderr": stdout} def _get_popen_args(self, host: str, port: int) -> list: - if callable(self.server_entry_point): + if callable(self.entry_point): path = str(pathlib.Path(__file__).parent / "panel_serve_render_fn.py") else: - path = pathlib.Path(self.server_entry_point) + path = pathlib.Path(self.entry_point) abs_path = str(path) - # The app is served at http://localhost:{port}/{flow}/{server_entry_point} + # The app is served at http://localhost:{port}/{flow}/{entry_point} # Lightning embeds http://localhost:{port}/{flow} but this redirects to the above and # seems to work fine. command = [ diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index 3bb621501af35..7aba5f2f20014 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -15,7 +15,7 @@ def test_stop_server_not_running(): """If the server is not running but stopped an Exception should be raised.""" - frontend = PanelFrontend(server_entry_point=Mock()) + frontend = PanelFrontend(entry_point=Mock()) with pytest.raises(RuntimeError, match="Server is not running."): frontend.stop_server() @@ -40,7 +40,7 @@ def run(self): # pylint: disable=arguments-differ def test_panel_frontend_start_stop_server(subprocess_mock): """Test that `PanelFrontend.start_server()` invokes subprocess.Popen with the right parameters.""" # Given - frontend = PanelFrontend(server_entry_point=_noop_render_fn) + frontend = PanelFrontend(entry_point=_noop_render_fn) frontend.flow = MockFlow() # When frontend.start_server(host="hostname", port=1111) @@ -97,20 +97,20 @@ def _call_me(state): "LIGHTNING_RENDER_PORT": "61896", }, ) -def test_panel_wrapper_calls_server_entry_point(*_): - """Run the panel_serve_server_entry_point.""" +def test_panel_wrapper_calls_entry_point(*_): + """Run the panel_serve_entry_point.""" runpy.run_module("lightning_app.frontend.panel.panel_serve_render_fn") def test_method_exception(): - """The PanelFrontend does not support server_entry_point being a method and should raise an Exception.""" + """The PanelFrontend does not support entry_point being a method and should raise an Exception.""" class _DummyClass: def _render_fn(self): pass with pytest.raises(TypeError, match="being a method"): - PanelFrontend(server_entry_point=_DummyClass()._render_fn) + PanelFrontend(entry_point=_DummyClass()._render_fn) def test_open_close_log_files(): From 9022b19ff5f6be7d5f5b27f98ed90cde7edad921 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 13 Aug 2022 04:56:54 +0000 Subject: [PATCH 049/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/panel/panel_frontend.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 54cbe15a47ee5..bf1670e61603b 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -82,8 +82,7 @@ def __init__(self, entry_point: Callable | str): if inspect.ismethod(entry_point): raise TypeError( - "The `PanelFrontend` doesn't support `entry_point` being a method. " - "Please, use a pure function." + "The `PanelFrontend` doesn't support `entry_point` being a method. " "Please, use a pure function." ) self.entry_point = entry_point From 94cc189d0f66f2a08f5666917ed83e3299107ef2 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sat, 13 Aug 2022 04:59:12 +0000 Subject: [PATCH 050/103] remove duplicate code --- src/lightning_app/frontend/streamlit_base.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/lightning_app/frontend/streamlit_base.py b/src/lightning_app/frontend/streamlit_base.py index af2a8314e07bd..2a75a4e475f44 100644 --- a/src/lightning_app/frontend/streamlit_base.py +++ b/src/lightning_app/frontend/streamlit_base.py @@ -9,6 +9,7 @@ from lightning_app.core.flow import LightningFlow from lightning_app.utilities.app_helpers import StreamLitStatePlugin from lightning_app.utilities.state import AppState +from lightning_app.frontend.utilities.utils import _reduce_to_flow_scope app_state = AppState(plugin=StreamLitStatePlugin()) @@ -20,19 +21,9 @@ def _get_render_fn_from_environment() -> Callable: return getattr(module, render_fn_name) -def _app_state_to_flow_scope(state: AppState, flow: Union[str, LightningFlow]) -> AppState: - """Returns a new AppState with the scope reduced to the given flow, as if the given flow as the root.""" - flow_name = flow.name if isinstance(flow, LightningFlow) else flow - flow_name_parts = flow_name.split(".")[1:] # exclude root - flow_state = state - for part in flow_name_parts: - flow_state = getattr(flow_state, part) - return flow_state - - def main(): # Fetch the information of which flow attaches to this streamlit instance - flow_state = _app_state_to_flow_scope(app_state, flow=os.environ["LIGHTNING_FLOW_NAME"]) + flow_state = _reduce_to_flow_scope(app_state, flow=os.environ["LIGHTNING_FLOW_NAME"]) # Call the provided render function. # Pass it the state, scoped to the current flow. From 28980957e36b5a65b4f4b83b43543dbc4fbf980a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 13 Aug 2022 05:02:06 +0000 Subject: [PATCH 051/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/streamlit_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/streamlit_base.py b/src/lightning_app/frontend/streamlit_base.py index 2a75a4e475f44..4d64f1c3cb24a 100644 --- a/src/lightning_app/frontend/streamlit_base.py +++ b/src/lightning_app/frontend/streamlit_base.py @@ -7,9 +7,9 @@ from typing import Callable, Union from lightning_app.core.flow import LightningFlow +from lightning_app.frontend.utilities.utils import _reduce_to_flow_scope from lightning_app.utilities.app_helpers import StreamLitStatePlugin from lightning_app.utilities.state import AppState -from lightning_app.frontend.utilities.utils import _reduce_to_flow_scope app_state = AppState(plugin=StreamLitStatePlugin()) From 41ee1393549986f5ed5a0f19a43baac96adc0a3d Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sat, 13 Aug 2022 05:16:24 +0000 Subject: [PATCH 052/103] move has_panel_auto_reload --- .../frontend/panel/panel_frontend.py | 24 ++++++++------ src/lightning_app/frontend/utilities/utils.py | 10 ------ .../frontend/panel/test_panel_frontend.py | 29 ++++++++++++++++- .../frontend/utilities/test_utils.py | 31 ------------------- 4 files changed, 42 insertions(+), 52 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index bf1670e61603b..33916cdfde404 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -3,24 +3,28 @@ import inspect import logging +import os import pathlib import subprocess import sys from typing import Callable, TextIO from lightning_app.frontend.frontend import Frontend -from lightning_app.frontend.utilities.utils import ( - get_allowed_hosts, - get_frontend_environment, - has_panel_autoreload, - is_running_locally, -) +from lightning_app.frontend.utilities.utils import get_allowed_hosts, get_frontend_environment, is_running_locally from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile _logger = logging.getLogger("PanelFrontend") +def has_panel_autoreload() -> bool: + """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. + + Please note the casing of value does not matter + """ + return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] + + class PanelFrontend(Frontend): """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. @@ -82,16 +86,16 @@ def __init__(self, entry_point: Callable | str): if inspect.ismethod(entry_point): raise TypeError( - "The `PanelFrontend` doesn't support `entry_point` being a method. " "Please, use a pure function." + "The `PanelFrontend` doesn't support `entry_point` being a method. Please, use a pure function." ) self.entry_point = entry_point self._process: None | subprocess.Popen = None self._log_files: dict[str, TextIO] = {} - _logger.debug(f"PanelFrontend Frontend with {entry_point} is initialized.") + _logger.debug("PanelFrontend Frontend with %s is initialized.", entry_point) def start_server(self, host: str, port: int) -> None: - _logger.debug(f"PanelFrontend starting server on {host}:{port}") + _logger.debug("PanelFrontend starting server on %s:%s", host, port) # 1: Prepare environment variables and arguments. env = get_frontend_environment( @@ -156,5 +160,5 @@ def _get_popen_args(self, host: str, port: int) -> list: ] if has_panel_autoreload(): command.append("--autoreload") - _logger.debug(f"PanelFrontend command {command}") + _logger.debug("PanelFrontend command %s", command) return command diff --git a/src/lightning_app/frontend/utilities/utils.py b/src/lightning_app/frontend/utilities/utils.py index cdc9ca874a62b..557fe611c0a3d 100644 --- a/src/lightning_app/frontend/utilities/utils.py +++ b/src/lightning_app/frontend/utilities/utils.py @@ -1,6 +1,4 @@ """Utility functions for lightning Frontends.""" -# TODO: Refactor stream_lit and streamlit_base to use this functionality. - from __future__ import annotations import inspect @@ -46,14 +44,6 @@ def get_allowed_hosts() -> str: return "*" -def has_panel_autoreload() -> bool: - """Returns True if the PANEL_AUTORELOAD environment variable is set to 'yes' or 'true'. - - Please note the casing of value does not matter - """ - return os.environ.get("PANEL_AUTORELOAD", "no").lower() in ["yes", "y", "true"] - - def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index 7aba5f2f20014..2cbc1aa2eea29 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -9,7 +9,7 @@ import pytest from lightning_app import LightningFlow -from lightning_app.frontend.panel import panel_serve_render_fn, PanelFrontend +from lightning_app.frontend.panel import has_panel_autoreload, panel_serve_render_fn, PanelFrontend from lightning_app.utilities.state import AppState @@ -134,3 +134,30 @@ def test_open_close_log_files(): # We can close even if not open frontend._close_log_files() + + +@pytest.mark.parametrize( + ["value", "expected"], + ( + ("Yes", True), + ("yes", True), + ("YES", True), + ("Y", True), + ("y", True), + ("True", True), + ("true", True), + ("TRUE", True), + ("No", False), + ("no", False), + ("NO", False), + ("N", False), + ("n", False), + ("False", False), + ("false", False), + ("FALSE", False), + ), +) +def test_has_panel_autoreload(value, expected): + """We can get and set autoreload via the environment variable PANEL_AUTORELOAD.""" + with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): + assert has_panel_autoreload() == expected diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/utilities/test_utils.py index 4a9271b3c63c3..604b842a02e7d 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/utilities/test_utils.py @@ -3,13 +3,10 @@ import os from unittest import mock -import pytest - from lightning_app.frontend.utilities.utils import ( get_flow_state, get_frontend_environment, get_render_fn_from_environment, - has_panel_autoreload, is_running_locally, ) from lightning_app.utilities.state import AppState @@ -63,34 +60,6 @@ def test_get_frontend_environment_file(): assert env["LIGHTNING_RENDER_FILE"] == "app_panel.py" assert env["LIGHTNING_RENDER_PORT"] == "1234" - -@pytest.mark.parametrize( - ["value", "expected"], - ( - ("Yes", True), - ("yes", True), - ("YES", True), - ("Y", True), - ("y", True), - ("True", True), - ("true", True), - ("TRUE", True), - ("No", False), - ("no", False), - ("NO", False), - ("N", False), - ("n", False), - ("False", False), - ("false", False), - ("FALSE", False), - ), -) -def test_has_panel_autoreload(value, expected): - """We can get and set autoreload via the environment variable PANEL_AUTORELOAD.""" - with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): - assert has_panel_autoreload() == expected - - @mock.patch.dict(os.environ, clear=True) def test_is_running_locally() -> bool: """We can determine if lightning is running locally.""" From 866ac06729fdde513d17cec66a789e5a7a9d326a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 13 Aug 2022 05:18:37 +0000 Subject: [PATCH 053/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_app/frontend/utilities/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/utilities/test_utils.py index 604b842a02e7d..8e4ce627447c6 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/utilities/test_utils.py @@ -60,6 +60,7 @@ def test_get_frontend_environment_file(): assert env["LIGHTNING_RENDER_FILE"] == "app_panel.py" assert env["LIGHTNING_RENDER_PORT"] == "1234" + @mock.patch.dict(os.environ, clear=True) def test_is_running_locally() -> bool: """We can determine if lightning is running locally.""" From 7334a16297f27b958462dbbbc1529e687eac25c5 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sat, 13 Aug 2022 06:34:27 +0000 Subject: [PATCH 054/103] fix imports --- tests/tests_app/frontend/panel/test_panel_frontend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index 2cbc1aa2eea29..0a05b867b5d52 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -9,7 +9,8 @@ import pytest from lightning_app import LightningFlow -from lightning_app.frontend.panel import has_panel_autoreload, panel_serve_render_fn, PanelFrontend +from lightning_app.frontend.panel import panel_serve_render_fn, PanelFrontend +from lightning_app.frontend.panel.panel_frontend import has_panel_autoreload from lightning_app.utilities.state import AppState From f77edf1397abb26ac3181e6df1eb5fda8ec134be Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:12:20 -0700 Subject: [PATCH 055/103] Update src/lightning_app/CHANGELOG.md --- src/lightning_app/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/CHANGELOG.md b/src/lightning_app/CHANGELOG.md index b7814aeca1666..deb77a543a8df 100644 --- a/src/lightning_app/CHANGELOG.md +++ b/src/lightning_app/CHANGELOG.md @@ -26,7 +26,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Add support for printing application logs using CLI `lightning show logs [components]` ([#13634](https://github.com/Lightning-AI/lightning/pull/13634)) -- Adds `PanelFrontend` to easily create complex UI in python ([#13531](https://github.com/Lightning-AI/lightning/pull/13531)) +- Adds `PanelFrontend` to easily create complex UI in Python ([#13531](https://github.com/Lightning-AI/lightning/pull/13531)) - Add support for `Lightning API` through the `configure_api` hook on the Lightning Flow and the `Post`, `Get`, `Delete`, `Put` HttpMethods ([#13945](https://github.com/Lightning-AI/lightning/pull/13945)) From c62256236c37c46456e5358de981248ba82aeded Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:13:07 -0700 Subject: [PATCH 056/103] Update src/lightning_app/frontend/panel/__init__.py --- src/lightning_app/frontend/panel/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/panel/__init__.py b/src/lightning_app/frontend/panel/__init__.py index d28eccabed9fb..501e98bc3d634 100644 --- a/src/lightning_app/frontend/panel/__init__.py +++ b/src/lightning_app/frontend/panel/__init__.py @@ -1,4 +1,4 @@ -"""The PanelFrontend and AppStateWatcher makes it easy to create lightning apps with the Panel data app +"""The PanelFrontend and AppStateWatcher make it easy to create Lightning Apps with the Panel data app framework.""" from lightning_app.frontend.panel.panel_frontend import PanelFrontend from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher From ec43faff426d76d32c5497c4983a44c77c978dce Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:23:27 -0700 Subject: [PATCH 057/103] Update src/lightning_app/frontend/panel/panel_frontend.py --- src/lightning_app/frontend/panel/panel_frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 33916cdfde404..3f41baf7f1949 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -68,7 +68,7 @@ def configure_layout(self): app = L.LightningApp(LitApp()) - You can start the lightning server with Panel autoreload by setting the `PANEL_AUTORELOAD` + You can start the Lightning server with Panel autoreload by setting the `PANEL_AUTORELOAD` environment variable to 'yes': `AUTORELOAD=yes lightning run app app_basic.py`. Args: From cc91fde20c5f1987c1dc3ffe042d193c79fb42f3 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:23:37 -0700 Subject: [PATCH 058/103] Update src/lightning_app/frontend/utilities/app_state_comm.py --- src/lightning_app/frontend/utilities/app_state_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 215eca1759e61..0b787d4765bde 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -43,7 +43,7 @@ async def update_fn(): while True: await websocket.recv() # Note: I have not seen use cases where the two lines below are needed - # Changing '< 0.2' to '< 1' makes the app very sluggish to the end user + # Changing '< 0.2' to '< 1' makes the App very sluggish to the end user # Also the implementation can make the app state get behind because only 1 update # is received per 0.2 second (or 1 second). # while (time.time() - last_updated) < 0.2: From 479f9d4f4a48208b95797903a747c0bbae90d77d Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:23:46 -0700 Subject: [PATCH 059/103] Update tests/tests_app/frontend/utilities/test_utils.py --- tests/tests_app/frontend/utilities/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/utilities/test_utils.py index 8e4ce627447c6..7cc90dfefa3f9 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/utilities/test_utils.py @@ -69,5 +69,5 @@ def test_is_running_locally() -> bool: @mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "127.0.0.1"}) def test_is_running_cloud() -> bool: - """We can determine if lightning is running in cloud.""" + """We can determine if Lightning is running in the cloud.""" assert not is_running_locally() From c2a1ff5282cb2de02f50ffd958b9f8a2602f80fd Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:23:58 -0700 Subject: [PATCH 060/103] Update tests/tests_app/frontend/utilities/test_utils.py --- tests/tests_app/frontend/utilities/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/utilities/test_utils.py index 7cc90dfefa3f9..e63652e7cefd6 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/utilities/test_utils.py @@ -63,7 +63,7 @@ def test_get_frontend_environment_file(): @mock.patch.dict(os.environ, clear=True) def test_is_running_locally() -> bool: - """We can determine if lightning is running locally.""" + """We can determine if Lightning is running locally.""" assert is_running_locally() From 4685e010aa5317f1dd8d956fabbf753bf2a80f68 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:24:11 -0700 Subject: [PATCH 061/103] Update tests/tests_app/frontend/utilities/test_app_state_watcher.py --- tests/tests_app/frontend/utilities/test_app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index aa480218fc556..d2e7a65787c5d 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,6 +1,6 @@ """The AppStateWatcher enables a Frontend to. -- subscribe to app state changes. +- subscribe to App state changes. - to access and change the app state. This is particularly useful for the PanelFrontend, but can be used by other Frontends too. From f4e6f0b4a1d03393bc00558ee215a8fd1558eebc Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:24:20 -0700 Subject: [PATCH 062/103] Update tests/tests_app/frontend/utilities/test_app_state_watcher.py --- tests/tests_app/frontend/utilities/test_app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index d2e7a65787c5d..a13e2128fbbc4 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -1,7 +1,7 @@ """The AppStateWatcher enables a Frontend to. - subscribe to App state changes. -- to access and change the app state. +- to access and change the App state. This is particularly useful for the PanelFrontend, but can be used by other Frontends too. """ From 52533434b055c8110d1c2bc9a78dcbf92b4dab46 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:24:30 -0700 Subject: [PATCH 063/103] Update tests/tests_app/frontend/panel/test_panel_serve_render_fn.py --- tests/tests_app/frontend/panel/test_panel_serve_render_fn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index ef9ca67881675..3bfc94d44b94c 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -1,4 +1,5 @@ -"""The panel_serve_render_fn_or_file file gets run by Python to lunch a Panel Server with Lightning. +"""The panel_serve_render_fn_or_file file gets run by Python to launch a Panel Server with Lightning. + These tests are for serving a render_fn function. """ From ebe849b161429e8a407786ea85b09b1da36c6009 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:24:46 -0700 Subject: [PATCH 064/103] Update tests/tests_app/frontend/utilities/test_app_state_comm.py --- tests/tests_app/frontend/utilities/test_app_state_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index 02667b3bd8ba8..0ab1f96955231 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -31,7 +31,7 @@ def test_watch_app_state(): # When watch_app_state(callback) - # Here we would like to send messages via the web socket + # Here we would like to send messages using the web socket # For testing the web socket is not started. See conftest.py # So we need to manually trigger _run_callbacks here _run_callbacks() From 02d66a4fbf310348b7b893901303de814c0e85a0 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:24:54 -0700 Subject: [PATCH 065/103] Update tests/tests_app/frontend/utilities/test_app_state_comm.py --- tests/tests_app/frontend/utilities/test_app_state_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index 0ab1f96955231..d793811aa613d 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -26,7 +26,7 @@ def test_get_ws_url_when_cloud(): @mock.patch.dict(os.environ, {"LIGHTNING_FLOW_NAME": "FLOW"}) def test_watch_app_state(): - """We can watch the app state and a callback function will be run when it changes.""" + """We can watch the App state and a callback function will be run when it changes.""" callback = mock.MagicMock() # When watch_app_state(callback) From fc790b3b6a83300be54994f7d24ba7c987da6e46 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:01 -0700 Subject: [PATCH 066/103] Update tests/tests_app/frontend/panel/test_panel_frontend.py --- tests/tests_app/frontend/panel/test_panel_frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/panel/test_panel_frontend.py b/tests/tests_app/frontend/panel/test_panel_frontend.py index 0a05b867b5d52..c31018378772c 100644 --- a/tests/tests_app/frontend/panel/test_panel_frontend.py +++ b/tests/tests_app/frontend/panel/test_panel_frontend.py @@ -159,6 +159,6 @@ def test_open_close_log_files(): ), ) def test_has_panel_autoreload(value, expected): - """We can get and set autoreload via the environment variable PANEL_AUTORELOAD.""" + """We can get and set autoreload using the environment variable PANEL_AUTORELOAD.""" with mock.patch.dict(os.environ, {"PANEL_AUTORELOAD": value}): assert has_panel_autoreload() == expected From 14f3e1af3da50d205b51aea0c603cf6269f01e1a Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:11 -0700 Subject: [PATCH 067/103] Update src/lightning_app/frontend/utilities/app_state_watcher.py --- src/lightning_app/frontend/utilities/app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 10a4fb3e3ba92..383887bec8912 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -1,7 +1,7 @@ """The AppStateWatcher enables a Frontend to. - subscribe to app state changes -- to access and change the app state. +- to access and change the App state. This is particularly useful for the PanelFrontend but can be used by other Frontends too. """ From bc0cab88375e35bb198ec797f7ce87da05aefaf2 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:19 -0700 Subject: [PATCH 068/103] Update src/lightning_app/frontend/utilities/app_state_watcher.py --- src/lightning_app/frontend/utilities/app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 383887bec8912..ffa10706ad8de 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -23,7 +23,7 @@ class AppStateWatcher(param.Parameterized): """The AppStateWatcher enables a Frontend to: - - Subscribe to any app state changes. + - Subscribe to any App state changes. - To access and change the app state from the UI. This is particularly useful for the PanelFrontend, but can be used by From f2331ba52e6b93a03867f6ff6d7e761dfec726f1 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:28 -0700 Subject: [PATCH 069/103] Update src/lightning_app/frontend/utilities/app_state_comm.py --- src/lightning_app/frontend/utilities/app_state_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 0b787d4765bde..86322bc7f5698 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -72,7 +72,7 @@ def watch_app_state(callback: Callable): """Start the process that serves the UI at the given hostname and port number. Arguments: - callback: A function to run when the app state changes. Must be thread safe. + callback: A function to run when the App state changes. Must be thread safe. Example: From 303d7cda79ceb87126b9f7f9961297a2f6552c82 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:39 -0700 Subject: [PATCH 070/103] Update src/lightning_app/frontend/utilities/app_state_watcher.py --- src/lightning_app/frontend/utilities/app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index ffa10706ad8de..5e6f5047b76ee 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -1,6 +1,6 @@ """The AppStateWatcher enables a Frontend to. -- subscribe to app state changes +- subscribe to App state changes - to access and change the App state. This is particularly useful for the PanelFrontend but can be used by other Frontends too. From e83f328d952ba87fb4897f880cbf79b2079b0ad9 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:48 -0700 Subject: [PATCH 071/103] Update src/lightning_app/frontend/utilities/app_state_watcher.py --- src/lightning_app/frontend/utilities/app_state_watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/utilities/app_state_watcher.py index 5e6f5047b76ee..fff4ec20695f5 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/utilities/app_state_watcher.py @@ -24,7 +24,7 @@ class AppStateWatcher(param.Parameterized): """The AppStateWatcher enables a Frontend to: - Subscribe to any App state changes. - - To access and change the app state from the UI. + - To access and change the App state from the UI. This is particularly useful for the PanelFrontend, but can be used by other Frontend's too. From 2ead2a1b5a5672aa3cd9df6adbcda81c2996123b Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:25:58 -0700 Subject: [PATCH 072/103] Update src/lightning_app/frontend/utilities/utils.py --- src/lightning_app/frontend/utilities/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/utils.py b/src/lightning_app/frontend/utilities/utils.py index 557fe611c0a3d..1e8b8c6d98b1e 100644 --- a/src/lightning_app/frontend/utilities/utils.py +++ b/src/lightning_app/frontend/utilities/utils.py @@ -71,7 +71,7 @@ def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: def is_running_locally() -> bool: - """Returns True if the lightning app is running locally. + """Returns True if the Lightning App is running locally. This function can be used to determine if the app is running locally and provide a better developer experience. """ From 3c13fb33fcf2bfda417a6789ddd9957e47b013da Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:26:07 -0700 Subject: [PATCH 073/103] Update src/lightning_app/frontend/utilities/utils.py --- src/lightning_app/frontend/utilities/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/utils.py b/src/lightning_app/frontend/utilities/utils.py index 1e8b8c6d98b1e..1aefcf2e4d9e4 100644 --- a/src/lightning_app/frontend/utilities/utils.py +++ b/src/lightning_app/frontend/utilities/utils.py @@ -73,6 +73,6 @@ def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: def is_running_locally() -> bool: """Returns True if the Lightning App is running locally. - This function can be used to determine if the app is running locally and provide a better developer experience. + This function can be used to determine if the App is running locally and provide a better developer experience. """ return "LIGHTNING_APP_STATE_URL" not in os.environ From d8a42303f6e12f819bc04aee18fc9c67993e7e13 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:26:16 -0700 Subject: [PATCH 074/103] Update src/lightning_app/utilities/state.py --- src/lightning_app/utilities/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/utilities/state.py b/src/lightning_app/utilities/state.py index 36c8927578392..378c3e20ec14e 100644 --- a/src/lightning_app/utilities/state.py +++ b/src/lightning_app/utilities/state.py @@ -66,7 +66,7 @@ def __init__( my_affiliation: Tuple[str, ...] = None, plugin: Optional[BaseStatePlugin] = None, ) -> None: - """The AppState class enable Frontend users to interact with their application state. + """The AppState class enables Frontend users to interact with their application state. When the state isn't defined, it would be pulled from the app REST API Server. If the state gets modified by the user, the new state would be sent to the API Server. From 1b97964c013d314999e461907780e3fb3ddc551f Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:26:24 -0700 Subject: [PATCH 075/103] Update tests/tests_app/frontend/utilities/test_app_state_comm.py --- tests/tests_app/frontend/utilities/test_app_state_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index d793811aa613d..271bfc8a4789a 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -1,4 +1,4 @@ -"""The watch_app_state function enables us to trigger a callback function when ever the app state changes.""" +"""The watch_app_state function enables us to trigger a callback function whenever the App state changes.""" import os from unittest import mock From aeeca10d24cce8f572f73934052d2504e58c14cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 13 Aug 2022 08:26:52 +0000 Subject: [PATCH 076/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/tests_app/frontend/panel/test_panel_serve_render_fn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 3bfc94d44b94c..06c38fe95e1d6 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -1,6 +1,5 @@ """The panel_serve_render_fn_or_file file gets run by Python to launch a Panel Server with Lightning. - These tests are for serving a render_fn function. """ import os From 9e33ad82598934bac063c93b771cf1f4ca19e114 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:27:58 -0700 Subject: [PATCH 077/103] Update src/lightning_app/frontend/panel/panel_serve_render_fn.py --- src/lightning_app/frontend/panel/panel_serve_render_fn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py index 3146f3efe12a4..d01bb6ac3a786 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -2,7 +2,8 @@ We will call the ``render_fn`` that the user provided to the PanelFrontend. -It requires the below environment variables to be set +It requires the following environment variables to be set + - LIGHTNING_RENDER_FUNCTION - LIGHTNING_RENDER_MODULE_FILE From c918d8aa754731659496cd6658d1a96704e91117 Mon Sep 17 00:00:00 2001 From: Laverne Henderson Date: Sat, 13 Aug 2022 01:28:25 -0700 Subject: [PATCH 078/103] Update src/lightning_app/frontend/utilities/app_state_comm.py --- src/lightning_app/frontend/utilities/app_state_comm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/utilities/app_state_comm.py index 86322bc7f5698..f7d9c01e7dc2d 100644 --- a/src/lightning_app/frontend/utilities/app_state_comm.py +++ b/src/lightning_app/frontend/utilities/app_state_comm.py @@ -44,7 +44,7 @@ async def update_fn(): await websocket.recv() # Note: I have not seen use cases where the two lines below are needed # Changing '< 0.2' to '< 1' makes the App very sluggish to the end user - # Also the implementation can make the app state get behind because only 1 update + # Also the implementation can cause the App state to lag behind because only 1 update # is received per 0.2 second (or 1 second). # while (time.time() - last_updated) < 0.2: # time.sleep(0.05) From 2cad3630bcc15c6862da942601b02024880be2c7 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Sun, 14 Aug 2022 05:02:40 +0000 Subject: [PATCH 079/103] remove unused imports --- src/lightning_app/frontend/streamlit_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lightning_app/frontend/streamlit_base.py b/src/lightning_app/frontend/streamlit_base.py index 4d64f1c3cb24a..9c0795c03b073 100644 --- a/src/lightning_app/frontend/streamlit_base.py +++ b/src/lightning_app/frontend/streamlit_base.py @@ -4,9 +4,8 @@ """ import os import pydoc -from typing import Callable, Union +from typing import Callable -from lightning_app.core.flow import LightningFlow from lightning_app.frontend.utilities.utils import _reduce_to_flow_scope from lightning_app.utilities.app_helpers import StreamLitStatePlugin from lightning_app.utilities.state import AppState @@ -22,6 +21,7 @@ def _get_render_fn_from_environment() -> Callable: def main(): + """Run the render_fn with the current flow_state""" # Fetch the information of which flow attaches to this streamlit instance flow_state = _reduce_to_flow_scope(app_state, flow=os.environ["LIGHTNING_FLOW_NAME"]) From 871802835c4b01ff13c9da9f3b6dd41589c4fd6e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Aug 2022 05:04:56 +0000 Subject: [PATCH 080/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/streamlit_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/streamlit_base.py b/src/lightning_app/frontend/streamlit_base.py index 9c0795c03b073..5e83494540bbf 100644 --- a/src/lightning_app/frontend/streamlit_base.py +++ b/src/lightning_app/frontend/streamlit_base.py @@ -21,7 +21,7 @@ def _get_render_fn_from_environment() -> Callable: def main(): - """Run the render_fn with the current flow_state""" + """Run the render_fn with the current flow_state.""" # Fetch the information of which flow attaches to this streamlit instance flow_state = _reduce_to_flow_scope(app_state, flow=os.environ["LIGHTNING_FLOW_NAME"]) From 35463ece575f3d2d1b1281ec93b5a8630b05c80c Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 11:19:49 +0200 Subject: [PATCH 081/103] revert code-owner change --- .github/CODEOWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cdbd85f443270..0b4692731bff9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -53,7 +53,6 @@ /src/lightning_app/runners/cloud.py @tchaton @hhsecond /src/lightning_app/testing @tchaton @manskx /src/lightning_app/__about__.py @nohalon @edenlightning @lantiga -src/lightning_app/frontend/panel @tchaton @MarcSkovMadsen # Examples /examples/app_* @tchaton @awaelchli @manskx @hhsecond From ed29d8f833ae67b182968e51de42e299ea1a9c6a Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 11:29:32 +0200 Subject: [PATCH 082/103] move images to s3 --- docs/source-app/workflows/add_web_ui/panel/basic.rst | 10 +++++----- .../workflows/add_web_ui/panel/intermediate.rst | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 26ff744374d85..2509952f29218 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -23,7 +23,7 @@ via `HoloViews`_, streaming and much more. - Panel is used by for example Rapids to power `CuxFilter`_, a CuDF based big data viz framework. - Panel can be deployed on your favorite server or cloud including `Lightning`_. -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-intro.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-intro.gif :alt: Example Panel App Example Panel App @@ -31,7 +31,7 @@ via `HoloViews`_, streaming and much more. Panel is **particularly well suited for lightning.ai apps** that needs to display live progress as the Panel server can react to state changes and asynchronously push messages from the server to the client via web socket communication. -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-streaming-intro.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-streaming-intro.gif :alt: Example Panel Streaming App Example Panel Streaming App @@ -109,7 +109,7 @@ Run the app locally to see it! The app should look like the below -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/images/panel-lightning/panel-lightning-basic.png +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-basic.png :alt: Basic Panel Lightning App Basic Panel Lightning App @@ -225,7 +225,7 @@ Try running the below PANEL_AUTORELOAD=yes lightning run app app.py -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-autoreload.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-autoreload.gif :alt: Basic Panel Lightning App with autoreload Basic Panel Lightning App with autoreload @@ -307,7 +307,7 @@ Finally run the app lightning run app app.py -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-theme.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-theme.gif :alt: Basic Panel Plotly Lightning App with theming Basic Panel Plotly Lightning App with theming diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index 2b32ce69775e6..3de7b34267af9 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -84,7 +84,7 @@ presses a button: app = L.LightningApp(LitApp()) -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-counter-from-frontend.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-counter-from-frontend.gif :alt: Panel Lightning App updating a counter from the frontend Panel Lightning App updating a counter from the frontend @@ -159,7 +159,7 @@ In this example, we update the ``count`` value of the component: app = L.LightningApp(LitApp()) -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-counter-from-component.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-counter-from-component.gif :alt: Panel Lightning App updating a counter from the component Panel Lightning App updating a counter from the component @@ -176,7 +176,7 @@ Tips & Tricks - Task Scheduling: Panel provides easy to use `task scheduling `_. You can use this to for example read and display files created by your components on a scheduled basis. - Terminal: Panel provides the `Xterm.js terminal `_ which can be used to display live logs from your components and allow you to provide a terminal interface to your component. -.. figure:: https://raw.githubusercontent.com/MarcSkovMadsen/awesome-panel-assets/master/videos/panel-lightning/panel-lightning-github-runner.gif +.. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-github-runner.gif :alt: Panel Lightning App running models on github Panel Lightning App running models on github From 5e12765ddb070285830cb5a9b759b0801dd260cf Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 11:36:54 +0200 Subject: [PATCH 083/103] move is_running_locally utility function --- src/lightning_app/frontend/panel/panel_frontend.py | 5 +++-- src/lightning_app/frontend/utilities/utils.py | 8 -------- src/lightning_app/utilities/cloud.py | 9 +++++++++ tests/tests_app/frontend/utilities/test_utils.py | 6 +++--- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 3f41baf7f1949..828a9087b722b 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -10,7 +10,8 @@ from typing import Callable, TextIO from lightning_app.frontend.frontend import Frontend -from lightning_app.frontend.utilities.utils import get_allowed_hosts, get_frontend_environment, is_running_locally +from lightning_app.frontend.utilities.utils import get_allowed_hosts, get_frontend_environment +from lightning_app.utilities.cloud import is_running_in_cloud from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -106,7 +107,7 @@ def start_server(self, host: str, port: int) -> None: ) command = self._get_popen_args(host, port) - if not is_running_locally(): + if is_running_in_cloud(): self._open_log_files() self._process = subprocess.Popen(command, env=env, **self._log_files) # pylint: disable=consider-using-with diff --git a/src/lightning_app/frontend/utilities/utils.py b/src/lightning_app/frontend/utilities/utils.py index 1aefcf2e4d9e4..28bc7d8faff06 100644 --- a/src/lightning_app/frontend/utilities/utils.py +++ b/src/lightning_app/frontend/utilities/utils.py @@ -68,11 +68,3 @@ def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: env["LIGHTNING_RENDER_MODULE_FILE"] = inspect.getmodule(render_fn_or_file).__file__ return env - - -def is_running_locally() -> bool: - """Returns True if the Lightning App is running locally. - - This function can be used to determine if the App is running locally and provide a better developer experience. - """ - return "LIGHTNING_APP_STATE_URL" not in os.environ diff --git a/src/lightning_app/utilities/cloud.py b/src/lightning_app/utilities/cloud.py index b320979a62028..11f32a8fb8e0b 100644 --- a/src/lightning_app/utilities/cloud.py +++ b/src/lightning_app/utilities/cloud.py @@ -1,3 +1,4 @@ +import os import warnings from lightning_cloud.openapi import V1Membership @@ -34,3 +35,11 @@ def _get_project(client: LightningClient, project_id: str = LIGHTNING_CLOUD_PROJ def _sigterm_flow_handler(*_, app: "lightning_app.LightningApp"): app.stage = AppStage.STOPPING + + +def is_running_in_cloud() -> bool: + """Returns True if the Lightning App is running in the cloud. + + This function can be used to determine if the App is running locally and provide a better developer experience. + """ + return "LIGHTNING_APP_STATE_URL" in os.environ diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/utilities/test_utils.py index e63652e7cefd6..8e7cd51744dfb 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/utilities/test_utils.py @@ -7,8 +7,8 @@ get_flow_state, get_frontend_environment, get_render_fn_from_environment, - is_running_locally, ) +from lightning_app.utilities.cloud import is_running_in_cloud from lightning_app.utilities.state import AppState @@ -64,10 +64,10 @@ def test_get_frontend_environment_file(): @mock.patch.dict(os.environ, clear=True) def test_is_running_locally() -> bool: """We can determine if Lightning is running locally.""" - assert is_running_locally() + assert is_running_in_cloud() @mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "127.0.0.1"}) def test_is_running_cloud() -> bool: """We can determine if Lightning is running in the cloud.""" - assert not is_running_locally() + assert not is_running_in_cloud() From 0df0fb54d0750e89568b71ebfd58efd32dad1107 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 11:59:05 +0200 Subject: [PATCH 084/103] move get_allowed_hosts --- src/lightning_app/frontend/panel/panel_frontend.py | 10 ++++++++-- src/lightning_app/frontend/utilities/utils.py | 6 ------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 828a9087b722b..3edafa39ae277 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -10,7 +10,7 @@ from typing import Callable, TextIO from lightning_app.frontend.frontend import Frontend -from lightning_app.frontend.utilities.utils import get_allowed_hosts, get_frontend_environment +from lightning_app.frontend.utilities.utils import get_frontend_environment from lightning_app.utilities.cloud import is_running_in_cloud from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -157,9 +157,15 @@ def _get_popen_args(self, host: str, port: int) -> list: "--prefix", self.flow.name, "--allow-websocket-origin", - get_allowed_hosts(), + _get_allowed_hosts(), ] if has_panel_autoreload(): command.append("--autoreload") _logger.debug("PanelFrontend command %s", command) return command + + +def _get_allowed_hosts() -> str: + """Returns a comma separated list of host[:port] that should be allowed to connect.""" + # TODO: Enable only lightning.ai domain in the cloud + return "*" diff --git a/src/lightning_app/frontend/utilities/utils.py b/src/lightning_app/frontend/utilities/utils.py index 28bc7d8faff06..151e6bef9ab8d 100644 --- a/src/lightning_app/frontend/utilities/utils.py +++ b/src/lightning_app/frontend/utilities/utils.py @@ -38,12 +38,6 @@ def get_flow_state(flow: str) -> AppState: return flow_state -def get_allowed_hosts() -> str: - """Returns a comma separated list of host[:port] that should be allowed to connect.""" - # TODO: Enable only lightning.ai domain in the cloud - return "*" - - def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. From 3e0a6068bfcb1dffa67251e32f958ecd3ee2643c Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 12:12:34 +0200 Subject: [PATCH 085/103] organize utilities --- src/lightning_app/frontend/panel/__init__.py | 2 +- .../{utilities => panel}/app_state_comm.py | 0 .../{utilities => panel}/app_state_watcher.py | 6 +++--- .../frontend/panel/panel_frontend.py | 4 ++-- .../frontend/panel/panel_serve_render_fn.py | 12 +++++++++--- src/lightning_app/frontend/streamlit_base.py | 2 +- .../frontend/utilities/__init__.py | 0 .../frontend/{utilities => }/utils.py | 19 ++++++------------- .../panel/test_panel_serve_render_fn.py | 2 +- .../frontend/utilities/test_app_state_comm.py | 2 +- .../utilities/test_app_state_watcher.py | 2 +- .../frontend/utilities/test_utils.py | 12 ++++++------ 12 files changed, 31 insertions(+), 32 deletions(-) rename src/lightning_app/frontend/{utilities => panel}/app_state_comm.py (100%) rename src/lightning_app/frontend/{utilities => panel}/app_state_watcher.py (94%) delete mode 100644 src/lightning_app/frontend/utilities/__init__.py rename src/lightning_app/frontend/{utilities => }/utils.py (70%) diff --git a/src/lightning_app/frontend/panel/__init__.py b/src/lightning_app/frontend/panel/__init__.py index 501e98bc3d634..831a6826ef1f8 100644 --- a/src/lightning_app/frontend/panel/__init__.py +++ b/src/lightning_app/frontend/panel/__init__.py @@ -1,6 +1,6 @@ """The PanelFrontend and AppStateWatcher make it easy to create Lightning Apps with the Panel data app framework.""" from lightning_app.frontend.panel.panel_frontend import PanelFrontend -from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher +from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher __all__ = ["PanelFrontend", "AppStateWatcher"] diff --git a/src/lightning_app/frontend/utilities/app_state_comm.py b/src/lightning_app/frontend/panel/app_state_comm.py similarity index 100% rename from src/lightning_app/frontend/utilities/app_state_comm.py rename to src/lightning_app/frontend/panel/app_state_comm.py diff --git a/src/lightning_app/frontend/utilities/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py similarity index 94% rename from src/lightning_app/frontend/utilities/app_state_watcher.py rename to src/lightning_app/frontend/panel/app_state_watcher.py index fff4ec20695f5..1dbef317fb6d8 100644 --- a/src/lightning_app/frontend/utilities/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -12,8 +12,8 @@ import param -from lightning_app.frontend.utilities.app_state_comm import watch_app_state -from lightning_app.frontend.utilities.utils import get_flow_state +from lightning_app.frontend.panel.app_state_comm import watch_app_state +from lightning_app.frontend.utils import _get_flow_state from lightning_app.utilities.imports import requires from lightning_app.utilities.state import AppState @@ -91,7 +91,7 @@ def _start_watching(self): def _get_flow_state(self) -> AppState: flow = os.environ["LIGHTNING_FLOW_NAME"] - return get_flow_state(flow) + return _get_flow_state(flow) def _update_flow_state(self): # Todo: Consider whether to only update if ._state changed diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 3edafa39ae277..d89ed898751be 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -10,7 +10,7 @@ from typing import Callable, TextIO from lightning_app.frontend.frontend import Frontend -from lightning_app.frontend.utilities.utils import get_frontend_environment +from lightning_app.frontend.utils import _get_frontend_environment from lightning_app.utilities.cloud import is_running_in_cloud from lightning_app.utilities.imports import requires from lightning_app.utilities.log import get_frontend_logfile @@ -99,7 +99,7 @@ def start_server(self, host: str, port: int) -> None: _logger.debug("PanelFrontend starting server on %s:%s", host, port) # 1: Prepare environment variables and arguments. - env = get_frontend_environment( + env = _get_frontend_environment( self.flow.name, self.entry_point, port, diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py index d01bb6ac3a786..6f4a3f9592a1e 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -16,15 +16,21 @@ """ import inspect import os +import pydoc -from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher -from lightning_app.frontend.utilities.utils import get_render_fn_from_environment +from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher + + +def _get_render_fn_from_environment(render_fn_name: str, render_fn_module_file: str) -> Callable: + """Returns the render_fn function to serve in the Frontend.""" + module = pydoc.importfile(render_fn_module_file) + return getattr(module, render_fn_name) def _get_render_fn(): render_fn_name = os.environ["LIGHTNING_RENDER_FUNCTION"] render_fn_module_file = os.environ["LIGHTNING_RENDER_MODULE_FILE"] - render_fn = get_render_fn_from_environment(render_fn_name, render_fn_module_file) + render_fn = _get_render_fn_from_environment(render_fn_name, render_fn_module_file) if inspect.signature(render_fn).parameters: def _render_fn_wrapper(): diff --git a/src/lightning_app/frontend/streamlit_base.py b/src/lightning_app/frontend/streamlit_base.py index 5e83494540bbf..c57ad2f9f9808 100644 --- a/src/lightning_app/frontend/streamlit_base.py +++ b/src/lightning_app/frontend/streamlit_base.py @@ -6,7 +6,7 @@ import pydoc from typing import Callable -from lightning_app.frontend.utilities.utils import _reduce_to_flow_scope +from lightning_app.frontend.utils import _reduce_to_flow_scope from lightning_app.utilities.app_helpers import StreamLitStatePlugin from lightning_app.utilities.state import AppState diff --git a/src/lightning_app/frontend/utilities/__init__.py b/src/lightning_app/frontend/utilities/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/lightning_app/frontend/utilities/utils.py b/src/lightning_app/frontend/utils.py similarity index 70% rename from src/lightning_app/frontend/utilities/utils.py rename to src/lightning_app/frontend/utils.py index 151e6bef9ab8d..1795445ef141f 100644 --- a/src/lightning_app/frontend/utilities/utils.py +++ b/src/lightning_app/frontend/utils.py @@ -3,19 +3,12 @@ import inspect import os -import pydoc from typing import Callable from lightning_app.core.flow import LightningFlow from lightning_app.utilities.state import AppState -def get_render_fn_from_environment(render_fn_name: str, render_fn_module_file: str) -> Callable: - """Returns the render_fn function to serve in the Frontend.""" - module = pydoc.importfile(render_fn_module_file) - return getattr(module, render_fn_name) - - def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppState: """Returns a new AppState with the scope reduced to the given flow.""" flow_name = flow.name if isinstance(flow, LightningFlow) else flow @@ -26,7 +19,7 @@ def _reduce_to_flow_scope(state: AppState, flow: str | LightningFlow) -> AppStat return flow_state -def get_flow_state(flow: str) -> AppState: +def _get_flow_state(flow: str) -> AppState: """Returns an AppState scoped to the current Flow. Returns: @@ -38,14 +31,14 @@ def get_flow_state(flow: str) -> AppState: return flow_state -def get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: +def _get_frontend_environment(flow: str, render_fn_or_file: Callable | str, port: int, host: str) -> os._Environ: """Returns an _Environ with the environment variables for serving a Frontend app set. Args: - flow (str): The name of the flow, for example root.lit_frontend - render_fn (Callable): A function to render - port (int): The port number, for example 54321 - host (str): The host, for example 'localhost' + flow: The name of the flow, for example root.lit_frontend + render_fn_or_file: A function to render + port: The port number, for example 54321 + host: The host, for example 'localhost' Returns: os._Environ: An environment diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 06c38fe95e1d6..08ec5b9a86feb 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -8,7 +8,7 @@ import pytest from lightning_app.frontend.panel.panel_serve_render_fn import _get_render_fn -from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher +from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher @pytest.fixture(autouse=True) diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/utilities/test_app_state_comm.py index 271bfc8a4789a..3766a1ccce564 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_comm.py +++ b/tests/tests_app/frontend/utilities/test_app_state_comm.py @@ -3,7 +3,7 @@ from unittest import mock from lightning_app.core.constants import APP_SERVER_PORT -from lightning_app.frontend.utilities.app_state_comm import _get_ws_url, _run_callbacks, watch_app_state +from lightning_app.frontend.panel.app_state_comm import _get_ws_url, _run_callbacks, watch_app_state FLOW_SUB = "lit_flow" FLOW = f"root.{FLOW_SUB}" diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/utilities/test_app_state_watcher.py index a13e2128fbbc4..25b99c8b25922 100644 --- a/tests/tests_app/frontend/utilities/test_app_state_watcher.py +++ b/tests/tests_app/frontend/utilities/test_app_state_watcher.py @@ -11,7 +11,7 @@ import pytest -from lightning_app.frontend.utilities.app_state_watcher import AppStateWatcher +from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher from lightning_app.utilities.state import AppState FLOW_SUB = "lit_flow" diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/utilities/test_utils.py index 8e7cd51744dfb..0162dc7f115d3 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/utilities/test_utils.py @@ -3,9 +3,9 @@ import os from unittest import mock -from lightning_app.frontend.utilities.utils import ( - get_flow_state, - get_frontend_environment, +from lightning_app.frontend.utils import ( + _get_flow_state, + _get_frontend_environment, get_render_fn_from_environment, ) from lightning_app.utilities.cloud import is_running_in_cloud @@ -15,7 +15,7 @@ def test_get_flow_state(flow_state_state: dict, flow): """We have a method to get an AppState scoped to the Flow state.""" # When - flow_state = get_flow_state(flow) + flow_state = _get_flow_state(flow) # Then assert isinstance(flow_state, AppState) assert flow_state._state == flow_state_state # pylint: disable=protected-access @@ -41,7 +41,7 @@ def some_fn(_): def test_get_frontend_environment_fn(): """We have a utility function to get the frontend render_fn environment.""" # When - env = get_frontend_environment(flow="root.lit_frontend", render_fn_or_file=some_fn, host="myhost", port=1234) + env = _get_frontend_environment(flow="root.lit_frontend", render_fn_or_file=some_fn, host="myhost", port=1234) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" @@ -53,7 +53,7 @@ def test_get_frontend_environment_fn(): def test_get_frontend_environment_file(): """We have a utility function to get the frontend render_fn environment.""" # When - env = get_frontend_environment(flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234) + env = _get_frontend_environment(flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" From 246eb745ac6ef478c796625efad486dfd474f42c Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 12:26:10 +0200 Subject: [PATCH 086/103] organize tests --- .../frontend/panel/panel_serve_render_fn.py | 1 + tests/tests_app/frontend/conftest.py | 2 +- .../test_app_state_comm.py | 0 .../test_app_state_watcher.py | 0 .../panel/test_panel_serve_render_fn.py | 16 +++++++++- .../frontend/{utilities => }/test_utils.py | 32 +------------------ tests/tests_app/utilities/test_cloud.py | 16 ++++++++++ 7 files changed, 34 insertions(+), 33 deletions(-) rename tests/tests_app/frontend/{utilities => panel}/test_app_state_comm.py (100%) rename tests/tests_app/frontend/{utilities => panel}/test_app_state_watcher.py (100%) rename tests/tests_app/frontend/{utilities => }/test_utils.py (60%) create mode 100644 tests/tests_app/utilities/test_cloud.py diff --git a/src/lightning_app/frontend/panel/panel_serve_render_fn.py b/src/lightning_app/frontend/panel/panel_serve_render_fn.py index 6f4a3f9592a1e..7aff3d5c3e601 100644 --- a/src/lightning_app/frontend/panel/panel_serve_render_fn.py +++ b/src/lightning_app/frontend/panel/panel_serve_render_fn.py @@ -17,6 +17,7 @@ import inspect import os import pydoc +from typing import Callable from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher diff --git a/tests/tests_app/frontend/conftest.py b/tests/tests_app/frontend/conftest.py index 3f0286ae42f7d..673fcf190508e 100644 --- a/tests/tests_app/frontend/conftest.py +++ b/tests/tests_app/frontend/conftest.py @@ -57,7 +57,7 @@ def do_nothing(): @pytest.fixture(autouse=True, scope="module") def mock_start_websocket(): """Avoid starting the websocket.""" - with mock.patch("lightning_app.frontend.utilities.app_state_comm._start_websocket", do_nothing): + with mock.patch("lightning_app.frontend.panel.app_state_comm._start_websocket", do_nothing): yield diff --git a/tests/tests_app/frontend/utilities/test_app_state_comm.py b/tests/tests_app/frontend/panel/test_app_state_comm.py similarity index 100% rename from tests/tests_app/frontend/utilities/test_app_state_comm.py rename to tests/tests_app/frontend/panel/test_app_state_comm.py diff --git a/tests/tests_app/frontend/utilities/test_app_state_watcher.py b/tests/tests_app/frontend/panel/test_app_state_watcher.py similarity index 100% rename from tests/tests_app/frontend/utilities/test_app_state_watcher.py rename to tests/tests_app/frontend/panel/test_app_state_watcher.py diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 08ec5b9a86feb..92ac436b09624 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -2,12 +2,13 @@ These tests are for serving a render_fn function. """ +import inspect import os from unittest import mock import pytest -from lightning_app.frontend.panel.panel_serve_render_fn import _get_render_fn +from lightning_app.frontend.panel.panel_serve_render_fn import _get_render_fn, _get_render_fn_from_environment from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher @@ -63,3 +64,16 @@ def test_get_view_fn_no_args(): """ result = _get_render_fn() assert result() == "no_args" + + +def render_fn_2(): + """Do nothing.""" + + +def test_get_render_fn_from_environment(): + """We have a method to get the render_fn from the environment.""" + # When + result = _get_render_fn_from_environment("render_fn_2", __file__) + # Then + assert result.__name__ == render_fn_2.__name__ + assert inspect.getmodule(result).__file__ == __file__ diff --git a/tests/tests_app/frontend/utilities/test_utils.py b/tests/tests_app/frontend/test_utils.py similarity index 60% rename from tests/tests_app/frontend/utilities/test_utils.py rename to tests/tests_app/frontend/test_utils.py index 0162dc7f115d3..941cda3c0d4f0 100644 --- a/tests/tests_app/frontend/utilities/test_utils.py +++ b/tests/tests_app/frontend/test_utils.py @@ -1,14 +1,6 @@ """We have some utility functions that can be used across frontends.""" -import inspect -import os -from unittest import mock -from lightning_app.frontend.utils import ( - _get_flow_state, - _get_frontend_environment, - get_render_fn_from_environment, -) -from lightning_app.utilities.cloud import is_running_in_cloud +from lightning_app.frontend.utils import _get_flow_state, _get_frontend_environment from lightning_app.utilities.state import AppState @@ -21,17 +13,6 @@ def test_get_flow_state(flow_state_state: dict, flow): assert flow_state._state == flow_state_state # pylint: disable=protected-access -def render_fn(): - """Do nothing.""" - - -def test_get_render_fn_from_environment(): - """We have a method to get the render_fn from the environment.""" - # When - result = get_render_fn_from_environment("render_fn", __file__) - # Then - assert result.__name__ == render_fn.__name__ - assert inspect.getmodule(result).__file__ == __file__ def some_fn(_): @@ -60,14 +41,3 @@ def test_get_frontend_environment_file(): assert env["LIGHTNING_RENDER_FILE"] == "app_panel.py" assert env["LIGHTNING_RENDER_PORT"] == "1234" - -@mock.patch.dict(os.environ, clear=True) -def test_is_running_locally() -> bool: - """We can determine if Lightning is running locally.""" - assert is_running_in_cloud() - - -@mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "127.0.0.1"}) -def test_is_running_cloud() -> bool: - """We can determine if Lightning is running in the cloud.""" - assert not is_running_in_cloud() diff --git a/tests/tests_app/utilities/test_cloud.py b/tests/tests_app/utilities/test_cloud.py new file mode 100644 index 0000000000000..573ec46106b84 --- /dev/null +++ b/tests/tests_app/utilities/test_cloud.py @@ -0,0 +1,16 @@ +import os +from unittest import mock + +from lightning_app.utilities.cloud import is_running_in_cloud + + +@mock.patch.dict(os.environ, clear=True) +def test_is_running_locally(): + """We can determine if Lightning is running locally.""" + assert not is_running_in_cloud() + + +@mock.patch.dict(os.environ, {"LIGHTNING_APP_STATE_URL": "127.0.0.1"}) +def test_is_running_cloud(): + """We can determine if Lightning is running in the cloud.""" + assert is_running_in_cloud() From 865fb25b44591ba67155dd89a185fa1627e6e520 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 12:28:09 +0200 Subject: [PATCH 087/103] formatting --- src/lightning_app/frontend/panel/__init__.py | 2 +- .../tests_app/frontend/panel/test_panel_serve_render_fn.py | 2 +- tests/tests_app/frontend/test_utils.py | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/lightning_app/frontend/panel/__init__.py b/src/lightning_app/frontend/panel/__init__.py index 831a6826ef1f8..ba76dd1dce1eb 100644 --- a/src/lightning_app/frontend/panel/__init__.py +++ b/src/lightning_app/frontend/panel/__init__.py @@ -1,6 +1,6 @@ """The PanelFrontend and AppStateWatcher make it easy to create Lightning Apps with the Panel data app framework.""" -from lightning_app.frontend.panel.panel_frontend import PanelFrontend from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher +from lightning_app.frontend.panel.panel_frontend import PanelFrontend __all__ = ["PanelFrontend", "AppStateWatcher"] diff --git a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py index 92ac436b09624..810367fe15934 100644 --- a/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py +++ b/tests/tests_app/frontend/panel/test_panel_serve_render_fn.py @@ -8,8 +8,8 @@ import pytest -from lightning_app.frontend.panel.panel_serve_render_fn import _get_render_fn, _get_render_fn_from_environment from lightning_app.frontend.panel.app_state_watcher import AppStateWatcher +from lightning_app.frontend.panel.panel_serve_render_fn import _get_render_fn, _get_render_fn_from_environment @pytest.fixture(autouse=True) diff --git a/tests/tests_app/frontend/test_utils.py b/tests/tests_app/frontend/test_utils.py index 941cda3c0d4f0..711eac464d830 100644 --- a/tests/tests_app/frontend/test_utils.py +++ b/tests/tests_app/frontend/test_utils.py @@ -13,8 +13,6 @@ def test_get_flow_state(flow_state_state: dict, flow): assert flow_state._state == flow_state_state # pylint: disable=protected-access - - def some_fn(_): """Be lazy!""" @@ -34,10 +32,11 @@ def test_get_frontend_environment_fn(): def test_get_frontend_environment_file(): """We have a utility function to get the frontend render_fn environment.""" # When - env = _get_frontend_environment(flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234) + env = _get_frontend_environment( + flow="root.lit_frontend", render_fn_or_file="app_panel.py", host="myhost", port=1234 + ) # Then assert env["LIGHTNING_FLOW_NAME"] == "root.lit_frontend" assert env["LIGHTNING_RENDER_ADDRESS"] == "myhost" assert env["LIGHTNING_RENDER_FILE"] == "app_panel.py" assert env["LIGHTNING_RENDER_PORT"] == "1234" - From d25287d8d6d3826b68e5e8d71763563038419160 Mon Sep 17 00:00:00 2001 From: Felonious-Spellfire Date: Sat, 13 Aug 2022 01:28:43 -0700 Subject: [PATCH 088/103] Review of content --- .../workflows/add_web_ui/panel/basic.rst | 63 ++++++++++--------- .../add_web_ui/panel/intermediate.rst | 30 +++++---- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/docs/source-app/workflows/add_web_ui/panel/basic.rst b/docs/source-app/workflows/add_web_ui/panel/basic.rst index 2509952f29218..695e6cdee2310 100644 --- a/docs/source-app/workflows/add_web_ui/panel/basic.rst +++ b/docs/source-app/workflows/add_web_ui/panel/basic.rst @@ -14,22 +14,25 @@ Add a web UI with Panel (basic) What is Panel? ************** -`Panel`_ and the `HoloViz`_ ecosystem provides unique and powerful -features such as big data viz via `DataShader`_, easy cross filtering -via `HoloViews`_, streaming and much more. +`Panel`_ and the `HoloViz`_ ecosystem provide unique and powerful +features such as big data visualization using `DataShader`_, easy cross filtering +using `HoloViews`_, streaming and much more. -- Panel is highly flexible and ties into the PyData and Jupyter ecosystems as you can develop in notebooks and use ipywidgets. You can also develop in .py files. -- Panel is one of the 4 most popular data app frameworks in Python with `more than 400.000 downloads a month `_. It's especially popular in the scientific community. -- Panel is used by for example Rapids to power `CuxFilter`_, a CuDF based big data viz framework. -- Panel can be deployed on your favorite server or cloud including `Lightning`_. +* Panel is highly flexible and ties into the PyData and Jupyter ecosystems as you can develop in notebooks and use ipywidgets. You can also develop in .py files. + +* Panel is one of the most popular data app frameworks in Python with `more than 400.000 downloads a month `_. It's especially popular in the scientific community. + +* Panel is used, for example, by Rapids to power `CuxFilter`_, a CuDF based big data visualization framework. + +* Panel can be deployed on your favorite server or cloud including `Lightning`_. .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-intro.gif :alt: Example Panel App Example Panel App -Panel is **particularly well suited for lightning.ai apps** that needs to display live progress as the Panel server can react -to state changes and asynchronously push messages from the server to the client via web socket communication. +Panel is **particularly well suited for Lightning Apps** that need to display live progress. This is because the Panel server can react +to state changes and asynchronously push messages from the server to the client using web socket communication. .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-streaming-intro.gif :alt: Example Panel Streaming App @@ -45,12 +48,12 @@ Install Panel with: ---- ********************* -Run a basic Panel app +Run a basic Panel App ********************* -In the next few sections, we'll build an app step-by-step. +In the next few sections, we'll build an App step-by-step. -First, create a file named ``app_panel.py`` with the app content: +First, create a file named ``app_panel.py`` with the App content: .. code:: python @@ -60,7 +63,7 @@ First, create a file named ``app_panel.py`` with the app content: pn.panel("Hello **Panel ⚡** World").servable() -Then, create a file named ``app.py`` with the following app content: +Then, create a file named ``app.py`` with the following App content: .. code:: python @@ -87,34 +90,34 @@ Then, create a file named ``app.py`` with the following app content: app = L.LightningApp(LitApp()) -add ``panel`` to your ``requirements.txt`` file: +Finally, add ``panel`` to your ``requirements.txt`` file: .. code:: bash echo 'panel' >> requirements.txt -This is a best practice to make apps reproducible. +.. note:: This is a best practice to make Apps reproducible. ---- *********** -Run the app +Run the App *********** -Run the app locally to see it! +Run the App locally: .. code:: bash lightning run app app.py -The app should look like the below +The App should look like this: .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-basic.png :alt: Basic Panel Lightning App Basic Panel Lightning App -Now, run it on the cloud as well: +Now, run it on the cloud: .. code:: bash @@ -126,7 +129,7 @@ Now, run it on the cloud as well: Step-by-step walk-through ************************* -In this section, we explain each part of this code in detail. +In this section, we explain each part of the code in detail. ---- @@ -145,11 +148,11 @@ Refer to the `Panel documentation `_ and `awesome-panel. ---- -1. Add Panel to a component +1. Add Panel to a Component ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Link this app to the Lightning App by using the ``PanelFrontend`` class which needs to be returned from -the ``configure_layout`` method of the Lightning component you want to connect to Panel. +the ``configure_layout`` method of the Lightning Component you want to connect to Panel. .. code:: python :emphasize-lines: 7-10 @@ -175,7 +178,7 @@ the ``configure_layout`` method of the Lightning component you want to connect t app = L.LightningApp(LitApp()) -The argument of the ``PanelFrontend`` class, points to the script, notebook or function that +The argument of the ``PanelFrontend`` class, points to the script, notebook, or function that runs your Panel app. ---- @@ -184,7 +187,7 @@ runs your Panel app. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The second step, is to tell the Root component in which tab to render this component's UI. -In this case, we render the ``LitPanel`` UI in the ``home`` tab of the application. +In this case, we render the ``LitPanel`` UI in the ``home`` tab of the app. .. code:: python :emphasize-lines: 16-17 @@ -216,10 +219,10 @@ Tips & Tricks 0. Use autoreload while developing ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To speed up your development workflow, you can run your lightning app with Panel **autoreload** by +To speed up your development workflow, you can run your Lightning App with Panel **autoreload** by setting the environment variable ``PANEL_AUTORELOAD`` to ``yes``. -Try running the below +Try running the following: .. code-block:: @@ -230,12 +233,12 @@ Try running the below Basic Panel Lightning App with autoreload -1. Theme your app +1. Theme your App ^^^^^^^^^^^^^^^^^ -To theme your app you, can use the lightning accent color ``#792EE5`` with the `FastListTemplate`_. +To theme your App you, can use the Lightning accent color ``#792EE5`` with the `FastListTemplate`_. -Try replacing the contents of ``app_panel.py`` with the below code. +Try replacing the contents of ``app_panel.py`` with the following: .. code:: bash @@ -301,7 +304,7 @@ Install some additional libraries and remember to add the dependencies to the `` echo 'plotly' >> requirements.txt echo 'pandas' >> requirements.txt -Finally run the app +Finally run the App .. code:: bash diff --git a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst index 3de7b34267af9..171f91d82c3b5 100644 --- a/docs/source-app/workflows/add_web_ui/panel/intermediate.rst +++ b/docs/source-app/workflows/add_web_ui/panel/intermediate.rst @@ -11,11 +11,11 @@ Add a web UI with Panel (intermediate) ---- ************************************** -Interact with the component from Panel +Interact with the Component from Panel ************************************** -The ``PanelFrontend`` enables user interactions with the Lightning App via widgets. -You can modify the state variables of a Lightning component via the ``AppStateWatcher``. +The ``PanelFrontend`` enables user interactions with the Lightning App using widgets. +You can modify the state variables of a Lightning Component using the ``AppStateWatcher``. For example, here we increase the ``count`` variable of the Lightning Component every time a user presses a button: @@ -92,13 +92,13 @@ presses a button: ---- ************************************ -Interact with Panel from a component +Interact with Panel from a Component ************************************ -To update the `PanelFrontend` from any Lightning component, update the property in the component. +To update the `PanelFrontend` from any Lightning Component, update the property in the Component. Make sure to call the ``run`` method from the parent component. -In this example, we update the ``count`` value of the component: +In this example, we update the ``count`` value of the Component: .. code:: python @@ -162,7 +162,7 @@ In this example, we update the ``count`` value of the component: .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-counter-from-component.gif :alt: Panel Lightning App updating a counter from the component - Panel Lightning App updating a counter from the component + Panel Lightning App updating a counter from the Component ---- @@ -170,16 +170,20 @@ In this example, we update the ``count`` value of the component: Tips & Tricks ************* -- Caching: Panel provides the easy to use ``pn.state.cache`` memory based, ``dict`` caching. If you are looking for something persistent try `DiskCache `_ its really powerful and simple to use. You can use it to communicate large amounts of data between the components and frontend(s). -- Notifications: Panel provides easy to use `notifications `_. You can for example use them to provide notifications about runs starting or ending. -- Tabulator Table: Panel provides the `Tabulator table `_ which features expandable rows. The table is useful to provide for example an overview of you runs. But you can dig into the details by clicking and expanding the row. -- Task Scheduling: Panel provides easy to use `task scheduling `_. You can use this to for example read and display files created by your components on a scheduled basis. -- Terminal: Panel provides the `Xterm.js terminal `_ which can be used to display live logs from your components and allow you to provide a terminal interface to your component. +* Caching: Panel provides the easy to use ``pn.state.cache`` memory based, ``dict`` caching. If you are looking for something persistent try `DiskCache `_ its really powerful and simple to use. You can use it to communicate large amounts of data between the components and frontend(s). + +* Notifications: Panel provides easy to use `notifications `_. You can for example use them to provide notifications about runs starting or ending. + +* Tabulator Table: Panel provides the `Tabulator table `_ which features expandable rows. The table is useful to provide for example an overview of you runs. But you can dig into the details by clicking and expanding the row. + +* Task Scheduling: Panel provides easy to use `task scheduling `_. You can use this to for example read and display files created by your components on a scheduled basis. + +* Terminal: Panel provides the `Xterm.js terminal `_ which can be used to display live logs from your components and allow you to provide a terminal interface to your component. .. figure:: https://pl-flash-data.s3.amazonaws.com/assets_lightning/docs/images/frontend/panel/panel-lightning-github-runner.gif :alt: Panel Lightning App running models on github - Panel Lightning App running models on github + Panel Lightning App running models on GitHub ---- From ccaf6139d2d9ff7f5e436367a88524f3bc985158 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Mon, 15 Aug 2022 15:53:41 +0200 Subject: [PATCH 089/103] reset cli changes --- tests/tests_app/cli/test_cmd_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_app/cli/test_cmd_install.py b/tests/tests_app/cli/test_cmd_install.py index 2ee39665f6875..0139bbc9c5501 100644 --- a/tests/tests_app/cli/test_cmd_install.py +++ b/tests/tests_app/cli/test_cmd_install.py @@ -263,9 +263,9 @@ def test_proper_url_parsing(): source_url, git_url, folder_name, git_sha = cmd_install._show_install_app_prompt( component_entry, app, org, True, resource_type="app" ) - assert folder_name == "video_search_react" + assert folder_name == "LAI-InVideo-search-App" # FixMe: this need to be updated after release with updated org rename - assert source_url == "https://github.com/PyTorchLightning/video_search_react" + assert source_url == "https://github.com/Lightning-AI/LAI-InVideo-search-App" assert "#ref" not in git_url assert git_sha From 29822b2f5b1cf59fd5af86913f1e8db927e64288 Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 23 Aug 2022 22:59:04 +0200 Subject: [PATCH 090/103] docs --- docs/source-app/conf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source-app/conf.py b/docs/source-app/conf.py index de2c737bf4fd2..6333774ee4e3b 100644 --- a/docs/source-app/conf.py +++ b/docs/source-app/conf.py @@ -22,7 +22,7 @@ _PATH_HERE = os.path.abspath(os.path.dirname(__file__)) _PATH_ROOT = os.path.realpath(os.path.join(_PATH_HERE, "..", "..")) -sys.path.insert(0, os.path.abspath(_PATH_ROOT)) +_PATH_REQUIRE = os.path.join(_PATH_ROOT, "requirements", "app") SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) @@ -306,10 +306,11 @@ def _package_list_from_file(file): PACKAGE_MAPPING = { "PyYAML": "yaml", } -MOCK_PACKAGES = [] +MOCK_PACKAGES = _package_list_from_file(os.path.join(_PATH_REQUIRE, "cloud.txt")) +MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_REQUIRE, "ui.txt")) if SPHINX_MOCK_REQUIREMENTS: # mock also base packages when we are on RTD since we don't install them there - MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_ROOT, "requirements.txt")) + MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_REQUIRE, "base.txt")) MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES] autodoc_mock_imports = MOCK_PACKAGES From cf27a294f63ce40504a3fc6233b666b7939584a9 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 2 Sep 2022 04:17:27 +0000 Subject: [PATCH 091/103] update docstring --- .../frontend/panel/app_state_watcher.py | 19 ++++----- .../frontend/panel/panel_frontend.py | 41 +++++++++++-------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index 1dbef317fb6d8..b4a8c7ca7dbd9 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -1,9 +1,9 @@ -"""The AppStateWatcher enables a Frontend to. +"""The `AppStateWatcher` enables a Frontend to - subscribe to App state changes - to access and change the App state. -This is particularly useful for the PanelFrontend but can be used by other Frontends too. +This is particularly useful for the `PanelFrontend` but can be used by other frontends too. """ from __future__ import annotations @@ -21,15 +21,16 @@ class AppStateWatcher(param.Parameterized): - """The AppStateWatcher enables a Frontend to: + """The `AppStateWatcher` enables a Frontend to: - Subscribe to any App state changes. - To access and change the App state from the UI. - This is particularly useful for the PanelFrontend, but can be used by - other Frontend's too. + This is particularly useful for the `PanelFrontend , but can be used by + other frontends too. - Example: + Example + ------- .. code-block:: python @@ -39,20 +40,18 @@ class AppStateWatcher(param.Parameterized): app.state.counter = 1 - @param.depends(app.param.state, watch=True) def update(state): print(f"The counter was updated to {state.counter}") - app.state.counter += 1 This would print ``The counter was updated to 2``. - The AppStateWatcher is built on top of Param which is a framework like dataclass, attrs and + The `AppStateWatcher is built on top of Param which is a framework like dataclass, attrs and Pydantic which additionally provides powerful and unique features for building reactive apps. - Please note the AppStateWatcher is a singleton, i.e. only one instance is instantiated + Please note the `AppStateWatcher` is a singleton, i.e. only one instance is instantiated """ state: AppState = param.ClassSelector( diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index d89ed898751be..1dcdbc056b1cd 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -27,17 +27,32 @@ def has_panel_autoreload() -> bool: class PanelFrontend(Frontend): - """The PanelFrontend enables you to serve Panel code as a Frontend for your LightningFlow. + """The `PanelFrontend` enables you to serve Panel code as a Frontend for your LightningFlow. - To use this frontend, you must first install the `panel` package: + Reference: https://lightning.ai/lightning-docs/workflows/add_web_ui/panel/ + + Parameters + ---------- + entry_point : The path to a .py or .ipynb file, or a pure function. + The file or function must contain your Panel code. + The function can optionally accept an `AppStateWatcher` argument. + + Raises + ------ + TypeError : Raised if the `entry_point`provided is a class method + + Example + ------- + + To use the `PanelFrontend`, you must first install the `panel` package: .. code-block:: bash pip install panel + + Create the files `panel_app_basic.py` and `app_basic.py` with the content below. - Example: - - `panel_app_basic.py` + **panel_app_basic.py** .. code-block:: python @@ -45,7 +60,7 @@ class PanelFrontend(Frontend): pn.panel("Hello **Panel ⚡** World").servable() - `app_basic.py` + **app_basic.py** .. code-block:: python @@ -69,20 +84,14 @@ def configure_layout(self): app = L.LightningApp(LitApp()) - You can start the Lightning server with Panel autoreload by setting the `PANEL_AUTORELOAD` - environment variable to 'yes': `AUTORELOAD=yes lightning run app app_basic.py`. - - Args: - entry_point: A pure function or the path to a .py or .ipynb file. - The function must be a pure function that contains your Panel code. - The function can optionally accept an `AppStateWatcher` argument. + Start the Lightning server with `lightning run app app_basic.py`. - Raises: - TypeError: Raised if the entry_point is a class method + For development you can get Panel autoreload by setting the `PANEL_AUTORELOAD` + environment variable to 'yes'. """ @requires("panel") - def __init__(self, entry_point: Callable | str): + def __init__(self, entry_point: str | Callable): super().__init__() if inspect.ismethod(entry_point): From 1501835f971d43e8d8b601c753875fc888702655 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 04:28:35 +0000 Subject: [PATCH 092/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/panel/app_state_watcher.py | 4 +++- src/lightning_app/frontend/panel/panel_frontend.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index b4a8c7ca7dbd9..aaadded2ec43a 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -1,4 +1,4 @@ -"""The `AppStateWatcher` enables a Frontend to +"""The `AppStateWatcher` enables a Frontend to. - subscribe to App state changes - to access and change the App state. @@ -40,10 +40,12 @@ class AppStateWatcher(param.Parameterized): app.state.counter = 1 + @param.depends(app.param.state, watch=True) def update(state): print(f"The counter was updated to {state.counter}") + app.state.counter += 1 This would print ``The counter was updated to 2``. diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 1dcdbc056b1cd..8496edc948159 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -33,14 +33,14 @@ class PanelFrontend(Frontend): Parameters ---------- - entry_point : The path to a .py or .ipynb file, or a pure function. + entry_point : The path to a .py or .ipynb file, or a pure function. The file or function must contain your Panel code. The function can optionally accept an `AppStateWatcher` argument. Raises ------ TypeError : Raised if the `entry_point`provided is a class method - + Example ------- @@ -49,7 +49,7 @@ class PanelFrontend(Frontend): .. code-block:: bash pip install panel - + Create the files `panel_app_basic.py` and `app_basic.py` with the content below. **panel_app_basic.py** @@ -87,7 +87,7 @@ def configure_layout(self): Start the Lightning server with `lightning run app app_basic.py`. For development you can get Panel autoreload by setting the `PANEL_AUTORELOAD` - environment variable to 'yes'. + environment variable to 'yes'. """ @requires("panel") From 2a6e9c1085e541368269b1d90043f9bb311a13da Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 2 Sep 2022 04:31:09 +0000 Subject: [PATCH 093/103] revert merge issues --- docs/source-app/conf.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source-app/conf.py b/docs/source-app/conf.py index 6333774ee4e3b..de2c737bf4fd2 100644 --- a/docs/source-app/conf.py +++ b/docs/source-app/conf.py @@ -22,7 +22,7 @@ _PATH_HERE = os.path.abspath(os.path.dirname(__file__)) _PATH_ROOT = os.path.realpath(os.path.join(_PATH_HERE, "..", "..")) -_PATH_REQUIRE = os.path.join(_PATH_ROOT, "requirements", "app") +sys.path.insert(0, os.path.abspath(_PATH_ROOT)) SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) @@ -306,11 +306,10 @@ def _package_list_from_file(file): PACKAGE_MAPPING = { "PyYAML": "yaml", } -MOCK_PACKAGES = _package_list_from_file(os.path.join(_PATH_REQUIRE, "cloud.txt")) -MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_REQUIRE, "ui.txt")) +MOCK_PACKAGES = [] if SPHINX_MOCK_REQUIREMENTS: # mock also base packages when we are on RTD since we don't install them there - MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_REQUIRE, "base.txt")) + MOCK_PACKAGES += _package_list_from_file(os.path.join(_PATH_ROOT, "requirements.txt")) MOCK_PACKAGES = [PACKAGE_MAPPING.get(pkg, pkg) for pkg in MOCK_PACKAGES] autodoc_mock_imports = MOCK_PACKAGES From 346c00ad30baa03a242d019715d81444b8425a52 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 04:34:24 +0000 Subject: [PATCH 094/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/panel/app_state_watcher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index b4a8c7ca7dbd9..aaadded2ec43a 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -1,4 +1,4 @@ -"""The `AppStateWatcher` enables a Frontend to +"""The `AppStateWatcher` enables a Frontend to. - subscribe to App state changes - to access and change the App state. @@ -40,10 +40,12 @@ class AppStateWatcher(param.Parameterized): app.state.counter = 1 + @param.depends(app.param.state, watch=True) def update(state): print(f"The counter was updated to {state.counter}") + app.state.counter += 1 This would print ``The counter was updated to 2``. From 6d81e188eabd28ae244188becf65da0542201da2 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 2 Sep 2022 04:37:09 +0000 Subject: [PATCH 095/103] fix merge issues --- .../frontend/panel/app_state_watcher.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index b4a8c7ca7dbd9..6c68b820d485b 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -10,17 +10,22 @@ import logging import os -import param - from lightning_app.frontend.panel.app_state_comm import watch_app_state from lightning_app.frontend.utils import _get_flow_state -from lightning_app.utilities.imports import requires +from lightning_app.utilities.imports import _is_param_available, requires from lightning_app.utilities.state import AppState _logger = logging.getLogger(__name__) -class AppStateWatcher(param.Parameterized): +if _is_param_available(): + from param import ClassSelector, edit_constant, Parameterized +else: + Parameterized = object + ClassSelector = dict + + +class AppStateWatcher(Parameterized): """The `AppStateWatcher` enables a Frontend to: - Subscribe to any App state changes. @@ -54,7 +59,7 @@ def update(state): Please note the `AppStateWatcher` is a singleton, i.e. only one instance is instantiated """ - state: AppState = param.ClassSelector( + state: AppState = ClassSelector( class_=AppState, constant=True, doc="The AppState holds the state of the app reduced to the scope of the Flow", @@ -69,7 +74,7 @@ def __new__(cls): @requires("param") def __init__(self): - # Its critical to initialize only once + # It is critical to initialize only once # See https://github.com/holoviz/param/issues/643 if not hasattr(self, "_initialized"): super().__init__(name="singleton") @@ -95,6 +100,6 @@ def _get_flow_state(self) -> AppState: def _update_flow_state(self): # Todo: Consider whether to only update if ._state changed # This might be much more performant. - with param.edit_constant(self): + with edit_constant(self): self.state = self._get_flow_state() _logger.debug("Requested App State.") From 4fd7733d178d2023f63ddb76cc594cdd886ff389 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 2 Sep 2022 04:38:52 +0000 Subject: [PATCH 096/103] remove spacing --- src/lightning_app/frontend/panel/app_state_watcher.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index f8e834ef029d8..29ee8291c47c3 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -1,4 +1,4 @@ -"""The `AppStateWatcher` enables a Frontend to. +"""The `AppStateWatcher` enables a Frontend to: - subscribe to App state changes - to access and change the App state. @@ -45,12 +45,10 @@ class AppStateWatcher(Parameterized): app.state.counter = 1 - @param.depends(app.param.state, watch=True) def update(state): print(f"The counter was updated to {state.counter}") - app.state.counter += 1 This would print ``The counter was updated to 2``. From d7866c3d46ae22d0e02b44572fca0916ea4c67f9 Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 2 Sep 2022 04:42:23 +0000 Subject: [PATCH 097/103] small docs improvements --- src/lightning_app/frontend/panel/app_state_watcher.py | 2 +- src/lightning_app/frontend/panel/panel_frontend.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index 29ee8291c47c3..2498d886489d8 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -53,7 +53,7 @@ def update(state): This would print ``The counter was updated to 2``. - The `AppStateWatcher is built on top of Param which is a framework like dataclass, attrs and + The `AppStateWatcher` is built on top of Param which is a framework like dataclass, attrs and Pydantic which additionally provides powerful and unique features for building reactive apps. Please note the `AppStateWatcher` is a singleton, i.e. only one instance is instantiated diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 8496edc948159..ccafabac9df94 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -87,7 +87,8 @@ def configure_layout(self): Start the Lightning server with `lightning run app app_basic.py`. For development you can get Panel autoreload by setting the `PANEL_AUTORELOAD` - environment variable to 'yes'. + environment variable to 'yes', i.e. run + `PANEL_AUTORELOAD=yes lightning run app app_basic.py` """ @requires("panel") From 82b49c98c79e58d312bf15d0c3fdb6b62284f4b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 04:44:32 +0000 Subject: [PATCH 098/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/panel/app_state_watcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index 2498d886489d8..a0a693318ff5e 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -45,10 +45,12 @@ class AppStateWatcher(Parameterized): app.state.counter = 1 + @param.depends(app.param.state, watch=True) def update(state): print(f"The counter was updated to {state.counter}") + app.state.counter += 1 This would print ``The counter was updated to 2``. From 392f41f6ed2b2075056c8754701bf7b242180895 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 22 Sep 2022 11:49:14 +0200 Subject: [PATCH 099/103] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- .../frontend/panel/panel_frontend.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index ccafabac9df94..8da3c64d738c7 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -31,18 +31,14 @@ class PanelFrontend(Frontend): Reference: https://lightning.ai/lightning-docs/workflows/add_web_ui/panel/ - Parameters - ---------- - entry_point : The path to a .py or .ipynb file, or a pure function. - The file or function must contain your Panel code. - The function can optionally accept an `AppStateWatcher` argument. - - Raises - ------ - TypeError : Raised if the `entry_point`provided is a class method - - Example - ------- + Args: + entry_point: The path to a .py or .ipynb file, or a pure function. The file or function must contain your Panel code. + The function can optionally accept an `AppStateWatcher` argument. + + Raises: + TypeError: Raised if the `entry_point`provided is a class method + + Example: To use the `PanelFrontend`, you must first install the `panel` package: From d11df1aa0908701e89cac1afe800cf7ec2b31a6e Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 22 Sep 2022 11:52:20 +0200 Subject: [PATCH 100/103] Apply suggestions from code review --- src/lightning_app/frontend/panel/app_state_watcher.py | 8 ++++---- src/lightning_app/frontend/panel/panel_frontend.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lightning_app/frontend/panel/app_state_watcher.py b/src/lightning_app/frontend/panel/app_state_watcher.py index a0a693318ff5e..f44f92106c377 100644 --- a/src/lightning_app/frontend/panel/app_state_watcher.py +++ b/src/lightning_app/frontend/panel/app_state_watcher.py @@ -1,9 +1,9 @@ -"""The `AppStateWatcher` enables a Frontend to: +"""The ``AppStateWatcher`` enables a Frontend to: - subscribe to App state changes - to access and change the App state. -This is particularly useful for the `PanelFrontend` but can be used by other frontends too. +This is particularly useful for the ``PanelFrontend`` but can be used by other frontends too. """ from __future__ import annotations @@ -55,10 +55,10 @@ def update(state): This would print ``The counter was updated to 2``. - The `AppStateWatcher` is built on top of Param which is a framework like dataclass, attrs and + The ``AppStateWatcher`` is built on top of Param, which is a framework like dataclass, attrs and Pydantic which additionally provides powerful and unique features for building reactive apps. - Please note the `AppStateWatcher` is a singleton, i.e. only one instance is instantiated + Please note the ``AppStateWatcher`` is a singleton, i.e., only one instance is instantiated """ state: AppState = ClassSelector( diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 8da3c64d738c7..e1d2c88be0f23 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -33,10 +33,10 @@ class PanelFrontend(Frontend): Args: entry_point: The path to a .py or .ipynb file, or a pure function. The file or function must contain your Panel code. - The function can optionally accept an `AppStateWatcher` argument. + The function can optionally accept an ``AppStateWatcher`` argument. Raises: - TypeError: Raised if the `entry_point`provided is a class method + TypeError: Raised if the ``entry_point`` provided is a class method Example: @@ -82,13 +82,13 @@ def configure_layout(self): Start the Lightning server with `lightning run app app_basic.py`. - For development you can get Panel autoreload by setting the `PANEL_AUTORELOAD` + For development you can get Panel autoreload by setting the ``PANEL_AUTORELOAD`` environment variable to 'yes', i.e. run - `PANEL_AUTORELOAD=yes lightning run app app_basic.py` + ``PANEL_AUTORELOAD=yes lightning run app app_basic.py`` """ @requires("panel") - def __init__(self, entry_point: str | Callable): + def __init__(self, entry_point: Union[str, Callable]): super().__init__() if inspect.ismethod(entry_point): From bcc1beb0922b496ab85457aaccd45885f70d3357 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 22 Sep 2022 11:52:53 +0200 Subject: [PATCH 101/103] Union --- src/lightning_app/frontend/panel/panel_frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index e1d2c88be0f23..02f8106d895f0 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -7,7 +7,7 @@ import pathlib import subprocess import sys -from typing import Callable, TextIO +from typing import Callable, TextIO, Union from lightning_app.frontend.frontend import Frontend from lightning_app.frontend.utils import _get_frontend_environment From e855904e1a701bd779738fe3e1fd1f2d79509c23 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Sep 2022 09:55:24 +0000 Subject: [PATCH 102/103] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/lightning_app/frontend/panel/panel_frontend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 02f8106d895f0..3234d08ef5200 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -88,7 +88,7 @@ def configure_layout(self): """ @requires("panel") - def __init__(self, entry_point: Union[str, Callable]): + def __init__(self, entry_point: str | Callable): super().__init__() if inspect.ismethod(entry_point): From eea859475024259b4e6cb4b2d5f02c2c0ccf6af5 Mon Sep 17 00:00:00 2001 From: awaelchli Date: Wed, 5 Oct 2022 01:49:09 +0200 Subject: [PATCH 103/103] precommit fixes --- src/lightning_app/frontend/panel/panel_frontend.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lightning_app/frontend/panel/panel_frontend.py b/src/lightning_app/frontend/panel/panel_frontend.py index 0a435f900f23f..16159631973ce 100644 --- a/src/lightning_app/frontend/panel/panel_frontend.py +++ b/src/lightning_app/frontend/panel/panel_frontend.py @@ -6,7 +6,7 @@ import pathlib import subprocess import sys -from typing import Callable, TextIO, Union +from typing import Callable, TextIO from lightning_app.frontend.frontend import Frontend from lightning_app.frontend.utils import _get_frontend_environment @@ -32,8 +32,8 @@ class PanelFrontend(Frontend): Reference: https://lightning.ai/lightning-docs/workflows/add_web_ui/panel/ Args: - entry_point: The path to a .py or .ipynb file, or a pure function. The file or function must contain your Panel code. - The function can optionally accept an ``AppStateWatcher`` argument. + entry_point: The path to a .py or .ipynb file, or a pure function. The file or function must contain your Panel + code. The function can optionally accept an ``AppStateWatcher`` argument. Raises: TypeError: Raised if the ``entry_point`` provided is a class method