From 786121d438599336ee81ecb2c704fc8c456a73b0 Mon Sep 17 00:00:00 2001 From: Philippe Moussalli Date: Wed, 31 Jan 2024 16:47:02 +0100 Subject: [PATCH 1/6] fix code inspection notebook --- src/fondant/pipeline/lightweight_component.py | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/fondant/pipeline/lightweight_component.py b/src/fondant/pipeline/lightweight_component.py index acd3816b..ddd82155 100644 --- a/src/fondant/pipeline/lightweight_component.py +++ b/src/fondant/pipeline/lightweight_component.py @@ -9,13 +9,14 @@ from importlib import metadata import pyarrow as pa +from IPython import get_ipython +from IPython.core.magics.code import extract_symbols from fondant.component import BaseComponent, Component from fondant.core.schema import Field, Type logger = logging.getLogger(__name__) - MIN_PYTHON_VERSION = (3, 8) MAX_PYTHON_VERSION = (3, 11) @@ -264,6 +265,33 @@ def consumes(cls) -> t.Optional[t.Dict[str, t.Dict[t.Any, t.Any]]]: return wrapper +def new_getfile(_object, _old_getfile=inspect.getfile): + if not inspect.isclass(_object): + return _old_getfile(_object) + + # Lookup by parent module (as in current inspect) + if hasattr(_object, "__module__"): + object_ = sys.modules.get(_object.__module__) + if hasattr(object_, "__file__"): + return object_.__file__ + + # If parent module is __main__, lookup by methods (NEW) + for name, member in inspect.getmembers(_object): + if ( + inspect.isfunction(member) + and _object.__qualname__ + "." + member.__name__ == member.__qualname__ + ): + return inspect.getfile(member) + + msg = f"Source for {_object!r} not found" + raise TypeError(msg) + + +def is_running_in_jupyter(): + shell = get_ipython().__class__.__name__ + return shell == "ZMQInteractiveShell" + + def build_python_script(component_cls: t.Type[Component]) -> str: """Build a self-contained python script for the provided component class, which will act as the `src/main.py` script to execute the component. @@ -281,7 +309,19 @@ def build_python_script(component_cls: t.Type[Component]) -> str: """, ) - component_source = inspect.getsource(component_cls) + if is_running_in_jupyter(): + inspect.getfile = new_getfile + component_source = "".join( + inspect.linecache.getlines( # type: ignore[attr-defined] + new_getfile(component_cls), + ), + ) + component_source = extract_symbols(component_source, component_cls.__name__)[0][ + 0 + ] + else: + component_source = inspect.getsource(component_cls) + component_source = textwrap.dedent(component_source) component_source_lines = component_source.split("\n") From 068ae12a486c98f28598780674cd6a5e1f8602f8 Mon Sep 17 00:00:00 2001 From: Philippe Moussalli Date: Wed, 31 Jan 2024 17:12:55 +0100 Subject: [PATCH 2/6] add ipython dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6b18c001..b538286b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ python = ">= 3.8, <3.12" fsspec = ">= 2023.4.0" importlib-resources = { version = ">= 1.3", python = "<3.9" } +ipython = ">= 8.21.0" jsonschema = ">= 4.18" pyarrow = ">= 11.0.0" pyyaml = ">= 5.3.1" From f0326c09c1c681a5d275605fe57eeb65918ce6c7 Mon Sep 17 00:00:00 2001 From: Philippe Moussalli Date: Wed, 31 Jan 2024 17:18:29 +0100 Subject: [PATCH 3/6] change method of env check --- pyproject.toml | 1 - src/fondant/pipeline/lightweight_component.py | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b538286b..6b18c001 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ python = ">= 3.8, <3.12" fsspec = ">= 2023.4.0" importlib-resources = { version = ">= 1.3", python = "<3.9" } -ipython = ">= 8.21.0" jsonschema = ">= 4.18" pyarrow = ">= 11.0.0" pyyaml = ">= 5.3.1" diff --git a/src/fondant/pipeline/lightweight_component.py b/src/fondant/pipeline/lightweight_component.py index ddd82155..a3d7cf97 100644 --- a/src/fondant/pipeline/lightweight_component.py +++ b/src/fondant/pipeline/lightweight_component.py @@ -9,8 +9,6 @@ from importlib import metadata import pyarrow as pa -from IPython import get_ipython -from IPython.core.magics.code import extract_symbols from fondant.component import BaseComponent, Component from fondant.core.schema import Field, Type @@ -287,9 +285,15 @@ def new_getfile(_object, _old_getfile=inspect.getfile): raise TypeError(msg) -def is_running_in_jupyter(): - shell = get_ipython().__class__.__name__ - return shell == "ZMQInteractiveShell" +def is_running_in_notebook(): + """Check if the code is running in a Jupyter notebook.""" + try: + from IPython import get_ipython + + shell = get_ipython().__class__.__name__ + return shell == "ZMQInteractiveShell" + except ModuleNotFoundError: + return False def build_python_script(component_cls: t.Type[Component]) -> str: @@ -309,7 +313,9 @@ def build_python_script(component_cls: t.Type[Component]) -> str: """, ) - if is_running_in_jupyter(): + if is_running_in_notebook(): + from IPython.core.magics.code import extract_symbols + inspect.getfile = new_getfile component_source = "".join( inspect.linecache.getlines( # type: ignore[attr-defined] From 0ca0d5aee1019913f9461c946dd430e223b189c1 Mon Sep 17 00:00:00 2001 From: Philippe Moussalli Date: Thu, 1 Feb 2024 12:08:06 +0100 Subject: [PATCH 4/6] implement PR feedback --- src/fondant/pipeline/lightweight_component.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/fondant/pipeline/lightweight_component.py b/src/fondant/pipeline/lightweight_component.py index a3d7cf97..1a39a708 100644 --- a/src/fondant/pipeline/lightweight_component.py +++ b/src/fondant/pipeline/lightweight_component.py @@ -285,13 +285,17 @@ def new_getfile(_object, _old_getfile=inspect.getfile): raise TypeError(msg) -def is_running_in_notebook(): - """Check if the code is running in a Jupyter notebook.""" +def is_running_interactively(): + """Check if the code is running in an interactive environment.""" try: from IPython import get_ipython shell = get_ipython().__class__.__name__ - return shell == "ZMQInteractiveShell" + return shell in [ + "ZMQInteractiveShell", + "TerminalInteractiveShell", + "PyDevTerminalInteractiveShell", + ] except ModuleNotFoundError: return False @@ -313,18 +317,20 @@ def build_python_script(component_cls: t.Type[Component]) -> str: """, ) - if is_running_in_notebook(): + if is_running_interactively(): from IPython.core.magics.code import extract_symbols - inspect.getfile = new_getfile component_source = "".join( inspect.linecache.getlines( # type: ignore[attr-defined] new_getfile(component_cls), ), ) - component_source = extract_symbols(component_source, component_cls.__name__)[0][ - 0 - ] + component_source = extract_symbols( + component_source, + component_cls.__name__, + ) + component_source = component_source[0][0] + else: component_source = inspect.getsource(component_cls) From 42b71236d1cf056263fdc97d4714feca6da89d11 Mon Sep 17 00:00:00 2001 From: Philippe Moussalli Date: Thu, 1 Feb 2024 13:22:22 +0100 Subject: [PATCH 5/6] support colab shell --- src/fondant/pipeline/lightweight_component.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fondant/pipeline/lightweight_component.py b/src/fondant/pipeline/lightweight_component.py index 1a39a708..e6b13485 100644 --- a/src/fondant/pipeline/lightweight_component.py +++ b/src/fondant/pipeline/lightweight_component.py @@ -292,6 +292,7 @@ def is_running_interactively(): shell = get_ipython().__class__.__name__ return shell in [ + "Shell", "ZMQInteractiveShell", "TerminalInteractiveShell", "PyDevTerminalInteractiveShell", From 6bdd0ed248e23bc3ee45860aa9c79dfb406d717d Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Mon, 5 Feb 2024 11:11:03 +0100 Subject: [PATCH 6/6] Simplify interactive check --- src/fondant/pipeline/lightweight_component.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/fondant/pipeline/lightweight_component.py b/src/fondant/pipeline/lightweight_component.py index e6b13485..4669e080 100644 --- a/src/fondant/pipeline/lightweight_component.py +++ b/src/fondant/pipeline/lightweight_component.py @@ -290,13 +290,7 @@ def is_running_interactively(): try: from IPython import get_ipython - shell = get_ipython().__class__.__name__ - return shell in [ - "Shell", - "ZMQInteractiveShell", - "TerminalInteractiveShell", - "PyDevTerminalInteractiveShell", - ] + return get_ipython() is not None except ModuleNotFoundError: return False