From cbf054eb7ac8fa75ecb101a43006dc8760fa18dc Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Tue, 21 May 2024 15:45:02 +0100 Subject: [PATCH 01/17] Lazy load subcommands Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 42 +++++++++++++++++++------------ kedro/framework/cli/lazy_group.py | 37 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 kedro/framework/cli/lazy_group.py diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 14a633a6cd..52b11c41d9 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -15,14 +15,8 @@ from kedro import __version__ as version from kedro.framework.cli import BRIGHT_BLACK, ORANGE -from kedro.framework.cli.catalog import catalog_cli from kedro.framework.cli.hooks import get_cli_hook_manager -from kedro.framework.cli.jupyter import jupyter_cli -from kedro.framework.cli.micropkg import micropkg_cli -from kedro.framework.cli.pipeline import pipeline_cli -from kedro.framework.cli.project import project_group -from kedro.framework.cli.registry import registry_cli -from kedro.framework.cli.starters import create_cli +from kedro.framework.cli.lazy_group import LazyGroup from kedro.framework.cli.utils import ( CONTEXT_SETTINGS, ENTRY_POINT_GROUPS, @@ -45,7 +39,24 @@ """ -@click.group(context_settings=CONTEXT_SETTINGS, name="Kedro") +@click.group( + cls=LazyGroup, + lazy_subcommands={ + "registry": "kedro.framework.cli.registry.registry", + "catalog": "kedro.framework.cli.catalog.catalog", + "ipython": "kedro.framework.cli.project.ipython", + "run": "kedro.framework.cli.project.run", + "micropkg": "kedro.framework.cli.micropkg.micropkg", + "package": "kedro.framework.cli.project.package", + "jupyter": "kedro.framework.cli.jupyter.jupyter", + "pipeline": "kedro.framework.cli.pipeline.pipeline", + "new": "kedro.framework.cli.starters.new", + "starters": "kedro.framework.cli.starters.starter", + # "project_group": "kedro.framework.cli.project.project_group", + }, + context_settings=CONTEXT_SETTINGS, + name="Kedro", +) @click.version_option(version, "--version", "-V", help="Show version and exit") def cli() -> None: # pragma: no cover """Kedro is a CLI for creating and using Kedro projects. For more @@ -125,7 +136,6 @@ def main( self._cli_hook_manager.hook.before_command_run( project_metadata=self._metadata, command_args=args ) - try: super().main( args=args, @@ -178,7 +188,7 @@ def global_groups(self) -> Sequence[click.MultiCommand]: combines them with the built-in ones (eventually overriding the built-in ones if they are redefined by plugins). """ - return [cli, create_cli, *load_entry_points("global")] + return [cli, *load_entry_points("global")] @property def project_groups(self) -> Sequence[click.MultiCommand]: @@ -193,12 +203,12 @@ def project_groups(self) -> Sequence[click.MultiCommand]: return [] built_in = [ - catalog_cli, - jupyter_cli, - pipeline_cli, - micropkg_cli, - project_group, - registry_cli, + # catalog_cli, + # jupyter_cli, + # pipeline_cli, + # micropkg_cli, + # project_group, + # registry_cli, ] plugins = load_entry_points("project") diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py new file mode 100644 index 0000000000..612c807c86 --- /dev/null +++ b/kedro/framework/cli/lazy_group.py @@ -0,0 +1,37 @@ +import importlib +import click + +class LazyGroup(click.Group): + def __init__(self, *args, lazy_subcommands=None, **kwargs): + super().__init__(*args, **kwargs) + # lazy_subcommands is a map of the form: + # + # {command-name} -> {module-name}.{command-object-name} + # + self.lazy_subcommands = lazy_subcommands or {} + + def list_commands(self, ctx): + base = super().list_commands(ctx) + lazy = sorted(self.lazy_subcommands.keys()) + return base + lazy + + def get_command(self, ctx, cmd_name): + if cmd_name in self.lazy_subcommands: + return self._lazy_load(cmd_name) + return super().get_command(ctx, cmd_name) + + def _lazy_load(self, cmd_name): + # lazily loading a command, first get the module name and attribute name + import_path = self.lazy_subcommands[cmd_name] + modname, cmd_object_name = import_path.rsplit(".", 1) + # do the import + mod = importlib.import_module(modname) + # get the Command object from that module + cmd_object = getattr(mod, cmd_object_name) + # check the result to make debugging easier + if not isinstance(cmd_object, click.BaseCommand): + raise ValueError( + f"Lazy loading of {import_path} failed by returning " + "a non-command object" + ) + return cmd_object \ No newline at end of file From f79fde3fd79cc8c55150fe33bc6ee5f3dc056766 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 24 Jun 2024 14:00:51 +0100 Subject: [PATCH 02/17] Test + lint Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 16 +++------------- kedro/framework/cli/lazy_group.py | 4 +++- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 52b11c41d9..2d7d1ef014 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -51,8 +51,7 @@ "jupyter": "kedro.framework.cli.jupyter.jupyter", "pipeline": "kedro.framework.cli.pipeline.pipeline", "new": "kedro.framework.cli.starters.new", - "starters": "kedro.framework.cli.starters.starter", - # "project_group": "kedro.framework.cli.project.project_group", + "starter": "kedro.framework.cli.starters.starter", }, context_settings=CONTEXT_SETTINGS, name="Kedro", @@ -202,15 +201,6 @@ def project_groups(self) -> Sequence[click.MultiCommand]: if not self._metadata: return [] - built_in = [ - # catalog_cli, - # jupyter_cli, - # pipeline_cli, - # micropkg_cli, - # project_group, - # registry_cli, - ] - plugins = load_entry_points("project") try: @@ -219,7 +209,7 @@ def project_groups(self) -> Sequence[click.MultiCommand]: except ModuleNotFoundError: # return only built-in commands and commands from plugins # (plugins can override built-in commands) - return [*built_in, *plugins] + return [*plugins] # fail badly if cli.py exists, but has no `cli` in it if not hasattr(project_cli, "cli"): @@ -229,7 +219,7 @@ def project_groups(self) -> Sequence[click.MultiCommand]: user_defined = project_cli.cli # return built-in commands, plugin commands and user defined commands # (overriding happens as follows built-in < plugins < cli.py) - return [*built_in, *plugins, user_defined] + return [*plugins, user_defined] def main() -> None: # pragma: no cover diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py index 612c807c86..08e0eae709 100644 --- a/kedro/framework/cli/lazy_group.py +++ b/kedro/framework/cli/lazy_group.py @@ -1,6 +1,8 @@ import importlib + import click + class LazyGroup(click.Group): def __init__(self, *args, lazy_subcommands=None, **kwargs): super().__init__(*args, **kwargs) @@ -34,4 +36,4 @@ def _lazy_load(self, cmd_name): f"Lazy loading of {import_path} failed by returning " "a non-command object" ) - return cmd_object \ No newline at end of file + return cmd_object From d53614a411bc755ced33012dcc5f2e9b9975ca7a Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 27 Jun 2024 15:12:05 +0100 Subject: [PATCH 03/17] Make overwriting of commands work Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 57 +++++++++++++++++++++++------------- kedro/framework/cli/utils.py | 48 +++++++++++++++--------------- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 2d7d1ef014..dbee3d55df 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -39,23 +39,7 @@ """ -@click.group( - cls=LazyGroup, - lazy_subcommands={ - "registry": "kedro.framework.cli.registry.registry", - "catalog": "kedro.framework.cli.catalog.catalog", - "ipython": "kedro.framework.cli.project.ipython", - "run": "kedro.framework.cli.project.run", - "micropkg": "kedro.framework.cli.micropkg.micropkg", - "package": "kedro.framework.cli.project.package", - "jupyter": "kedro.framework.cli.jupyter.jupyter", - "pipeline": "kedro.framework.cli.pipeline.pipeline", - "new": "kedro.framework.cli.starters.new", - "starter": "kedro.framework.cli.starters.starter", - }, - context_settings=CONTEXT_SETTINGS, - name="Kedro", -) +@click.group(context_settings=CONTEXT_SETTINGS, name="Kedro") @click.version_option(version, "--version", "-V", help="Show version and exit") def cli() -> None: # pragma: no cover """Kedro is a CLI for creating and using Kedro projects. For more @@ -95,6 +79,39 @@ def info() -> None: click.echo("No plugins installed") +@click.group( + context_settings=CONTEXT_SETTINGS, + cls=LazyGroup, + name="Kedro", + lazy_subcommands={ + "registry": "kedro.framework.cli.registry.registry", + "catalog": "kedro.framework.cli.catalog.catalog", + "ipython": "kedro.framework.cli.project.ipython", + "run": "kedro.framework.cli.project.run", + "micropkg": "kedro.framework.cli.micropkg.micropkg", + "package": "kedro.framework.cli.project.package", + "jupyter": "kedro.framework.cli.jupyter.jupyter", + "pipeline": "kedro.framework.cli.pipeline.pipeline", + }, +) +def project_group(): + pass + + +@click.group( + context_settings=CONTEXT_SETTINGS, + name="Kedro", + cls=LazyGroup, + lazy_subcommands={ + "new": "kedro.framework.cli.starters.new", + "starter": "kedro.framework.cli.starters.starter", + "info": "kedro.framework.cli.cli.info", + }, +) +def global_group(): + pass + + def _init_plugins() -> None: init_hooks = load_entry_points("init") for init_hook in init_hooks: @@ -187,7 +204,7 @@ def global_groups(self) -> Sequence[click.MultiCommand]: combines them with the built-in ones (eventually overriding the built-in ones if they are redefined by plugins). """ - return [cli, *load_entry_points("global")] + return [cli, global_group, *load_entry_points("global")] @property def project_groups(self) -> Sequence[click.MultiCommand]: @@ -209,7 +226,7 @@ def project_groups(self) -> Sequence[click.MultiCommand]: except ModuleNotFoundError: # return only built-in commands and commands from plugins # (plugins can override built-in commands) - return [*plugins] + return [*plugins, project_group] # fail badly if cli.py exists, but has no `cli` in it if not hasattr(project_cli, "cli"): @@ -219,7 +236,7 @@ def project_groups(self) -> Sequence[click.MultiCommand]: user_defined = project_cli.cli # return built-in commands, plugin commands and user defined commands # (overriding happens as follows built-in < plugins < cli.py) - return [*plugins, user_defined] + return [user_defined, *plugins, project_group] def main() -> None: # pragma: no cover diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index 2d298bd770..dde485189b 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -120,14 +120,13 @@ def __init__(self, *groups: tuple[str, Sequence[click.MultiCommand]]): for title, cli_list in groups ] sources = list(chain.from_iterable(cli_list for _, cli_list in self.groups)) - help_texts = [ cli.help for cli_collection in sources for cli in cli_collection.sources if cli.help ] - self._dedupe_commands(sources) + # self._dedupe_commands(sources) super().__init__( sources=sources, # type: ignore[arg-type] help="\n\n".join(help_texts), @@ -136,28 +135,28 @@ def __init__(self, *groups: tuple[str, Sequence[click.MultiCommand]]): self.params = sources[0].params self.callback = sources[0].callback - @staticmethod - def _dedupe_commands(cli_collections: Sequence[click.CommandCollection]) -> None: - """Deduplicate commands by keeping the ones from the last source - in the list. - """ - seen_names: set[str] = set() - for cli_collection in reversed(cli_collections): - for cmd_group in reversed(cli_collection.sources): - cmd_group.commands = { # type: ignore[attr-defined] - cmd_name: cmd - for cmd_name, cmd in cmd_group.commands.items() # type: ignore[attr-defined] - if cmd_name not in seen_names - } - seen_names |= cmd_group.commands.keys() # type: ignore[attr-defined] - - # remove empty command groups - for cli_collection in cli_collections: - cli_collection.sources = [ - cmd_group - for cmd_group in cli_collection.sources - if cmd_group.commands # type: ignore[attr-defined] - ] + # @staticmethod + # def _dedupe_commands(cli_collections: Sequence[click.CommandCollection]) -> None: + # """Deduplicate commands by keeping the ones from the last source + # in the list. + # """ + # seen_names: set[str] = set() + # for cli_collection in reversed(cli_collections): + # for cmd_group in reversed(cli_collection.sources): + # cmd_group.commands = { # type: ignore[attr-defined] + # cmd_name: cmd + # for cmd_name, cmd in cmd_group.commands.items() # type: ignore[attr-defined] + # if cmd_name not in seen_names + # } + # seen_names |= cmd_group.commands.keys() # type: ignore[attr-defined] + + # # remove empty command groups + # for cli_collection in cli_collections: + # cli_collection.sources = [ + # cmd_group + # for cmd_group in cli_collection.sources + # if cmd_group.commands # type: ignore[attr-defined] + # ] @staticmethod def _merge_same_name_collections( @@ -169,7 +168,6 @@ def _merge_same_name_collections( named_groups[group.name].append(group) # type: ignore[index] if group.help: helps[group.name].append(group.help) # type: ignore[index] - return [ click.CommandCollection( name=group_name, From 05ada823f12c376ad4a9507d80e63360a7f4c8db Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 28 Jun 2024 12:59:50 +0100 Subject: [PATCH 04/17] Some lint fixes Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 4 ++-- kedro/framework/cli/lazy_group.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index dbee3d55df..07a4f6cc41 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -94,7 +94,7 @@ def info() -> None: "pipeline": "kedro.framework.cli.pipeline.pipeline", }, ) -def project_group(): +def project_group() -> None: pass @@ -108,7 +108,7 @@ def project_group(): "info": "kedro.framework.cli.cli.info", }, ) -def global_group(): +def global_group() -> None: pass diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py index 08e0eae709..96d96c9e4e 100644 --- a/kedro/framework/cli/lazy_group.py +++ b/kedro/framework/cli/lazy_group.py @@ -4,7 +4,7 @@ class LazyGroup(click.Group): - def __init__(self, *args, lazy_subcommands=None, **kwargs): + def __init__(self, *args, lazy_subcommands: dict[str, str] | None = None, **kwargs): super().__init__(*args, **kwargs) # lazy_subcommands is a map of the form: # @@ -12,17 +12,17 @@ def __init__(self, *args, lazy_subcommands=None, **kwargs): # self.lazy_subcommands = lazy_subcommands or {} - def list_commands(self, ctx): + def list_commands(self, ctx: click.Context): base = super().list_commands(ctx) lazy = sorted(self.lazy_subcommands.keys()) return base + lazy - def get_command(self, ctx, cmd_name): + def get_command(self, ctx: click.Context, cmd_name: str): if cmd_name in self.lazy_subcommands: return self._lazy_load(cmd_name) return super().get_command(ctx, cmd_name) - def _lazy_load(self, cmd_name): + def _lazy_load(self, cmd_name: str): # lazily loading a command, first get the module name and attribute name import_path = self.lazy_subcommands[cmd_name] modname, cmd_object_name = import_path.rsplit(".", 1) From f38676ca4840b45c41ea664fb4dac2504273e024 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 28 Jun 2024 13:14:32 +0100 Subject: [PATCH 05/17] Some lint fixes Signed-off-by: Ankita Katiyar --- kedro/framework/cli/lazy_group.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py index 96d96c9e4e..c17521656c 100644 --- a/kedro/framework/cli/lazy_group.py +++ b/kedro/framework/cli/lazy_group.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import importlib import click From 6dde38ab94b35bcec3dd83d796e8b68dd8a755f8 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 28 Jun 2024 13:17:12 +0100 Subject: [PATCH 06/17] Some lint fixes Signed-off-by: Ankita Katiyar --- kedro/framework/cli/lazy_group.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py index c17521656c..3b2abfe5cb 100644 --- a/kedro/framework/cli/lazy_group.py +++ b/kedro/framework/cli/lazy_group.py @@ -14,17 +14,17 @@ def __init__(self, *args, lazy_subcommands: dict[str, str] | None = None, **kwar # self.lazy_subcommands = lazy_subcommands or {} - def list_commands(self, ctx: click.Context): - base = super().list_commands(ctx) + def list_commands(self, ctx: click.Context) -> list[str]: + base = list(super().list_commands(ctx)) lazy = sorted(self.lazy_subcommands.keys()) return base + lazy - def get_command(self, ctx: click.Context, cmd_name: str): - if cmd_name in self.lazy_subcommands: + def get_command(self, ctx: click.Context, cmd_name: str) -> click.BaseCommand | click.Command | None: # type: ignore[override] + if cmd_name in self.lazy_subcommands: return self._lazy_load(cmd_name) return super().get_command(ctx, cmd_name) - def _lazy_load(self, cmd_name: str): + def _lazy_load(self, cmd_name: str) -> click.BaseCommand: # lazily loading a command, first get the module name and attribute name import_path = self.lazy_subcommands[cmd_name] modname, cmd_object_name = import_path.rsplit(".", 1) From c6954e73d1f91fcb65a97b88b3e4bf86190f05d4 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 28 Jun 2024 13:28:31 +0100 Subject: [PATCH 07/17] Some lint fixes Signed-off-by: Ankita Katiyar --- kedro/framework/cli/lazy_group.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py index 3b2abfe5cb..b28d901666 100644 --- a/kedro/framework/cli/lazy_group.py +++ b/kedro/framework/cli/lazy_group.py @@ -19,8 +19,10 @@ def list_commands(self, ctx: click.Context) -> list[str]: lazy = sorted(self.lazy_subcommands.keys()) return base + lazy - def get_command(self, ctx: click.Context, cmd_name: str) -> click.BaseCommand | click.Command | None: # type: ignore[override] - if cmd_name in self.lazy_subcommands: + def get_command( + self, ctx: click.Context, cmd_name: str + ) -> click.BaseCommand | click.Command | None: # type: ignore[override] + if cmd_name in self.lazy_subcommands: return self._lazy_load(cmd_name) return super().get_command(ctx, cmd_name) From 63a4356e99e731cfeadb2f0000ca758dda06e172 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 28 Jun 2024 14:59:07 +0100 Subject: [PATCH 08/17] Fix some tests Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 11 +++--- tests/framework/cli/test_cli.py | 60 +++++++++++++++------------------ 2 files changed, 32 insertions(+), 39 deletions(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 07a4f6cc41..a7bdc0bf2e 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -94,7 +94,7 @@ def info() -> None: "pipeline": "kedro.framework.cli.pipeline.pipeline", }, ) -def project_group() -> None: +def project_commands() -> None: pass @@ -105,10 +105,9 @@ def project_group() -> None: lazy_subcommands={ "new": "kedro.framework.cli.starters.new", "starter": "kedro.framework.cli.starters.starter", - "info": "kedro.framework.cli.cli.info", }, ) -def global_group() -> None: +def global_commands() -> None: pass @@ -204,7 +203,7 @@ def global_groups(self) -> Sequence[click.MultiCommand]: combines them with the built-in ones (eventually overriding the built-in ones if they are redefined by plugins). """ - return [cli, global_group, *load_entry_points("global")] + return [cli, global_commands, *load_entry_points("global")] @property def project_groups(self) -> Sequence[click.MultiCommand]: @@ -226,7 +225,7 @@ def project_groups(self) -> Sequence[click.MultiCommand]: except ModuleNotFoundError: # return only built-in commands and commands from plugins # (plugins can override built-in commands) - return [*plugins, project_group] + return [*plugins, project_commands] # fail badly if cli.py exists, but has no `cli` in it if not hasattr(project_cli, "cli"): @@ -236,7 +235,7 @@ def project_groups(self) -> Sequence[click.MultiCommand]: user_defined = project_cli.cli # return built-in commands, plugin commands and user defined commands # (overriding happens as follows built-in < plugins < cli.py) - return [user_defined, *plugins, project_group] + return [user_defined, *plugins, project_commands] def main() -> None: # pragma: no cover diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index 857e71a859..5eaa6e643e 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -11,14 +11,13 @@ from kedro import KedroDeprecationWarning from kedro import __version__ as version from kedro.framework.cli import load_entry_points -from kedro.framework.cli.catalog import catalog_cli -from kedro.framework.cli.cli import KedroCLI, _init_plugins, cli -from kedro.framework.cli.jupyter import jupyter_cli -from kedro.framework.cli.micropkg import micropkg_cli -from kedro.framework.cli.pipeline import pipeline_cli -from kedro.framework.cli.project import project_group -from kedro.framework.cli.registry import registry_cli -from kedro.framework.cli.starters import create_cli +from kedro.framework.cli.cli import ( + KedroCLI, + _init_plugins, + cli, + global_commands, + project_commands, +) from kedro.framework.cli.utils import ( CommandCollection, KedroCliError, @@ -333,15 +332,8 @@ def test_project_commands_no_clipy(self, mocker, fake_metadata): ) kedro_cli = KedroCLI(fake_metadata.project_path) print(kedro_cli.project_groups) - assert len(kedro_cli.project_groups) == 6 - assert kedro_cli.project_groups == [ - catalog_cli, - jupyter_cli, - pipeline_cli, - micropkg_cli, - project_group, - registry_cli, - ] + assert len(kedro_cli.project_groups) == 1 + assert kedro_cli.project_groups == [project_commands] def test_project_commands_no_project(self, mocker, tmp_path): mocker.patch("kedro.framework.cli.cli._is_project", return_value=False) @@ -371,22 +363,23 @@ def test_project_commands_valid_clipy(self, mocker, fake_metadata): return_value=Module(cli=cli), ) kedro_cli = KedroCLI(fake_metadata.project_path) - assert len(kedro_cli.project_groups) == 7 + assert len(kedro_cli.project_groups) == 2 assert kedro_cli.project_groups == [ - catalog_cli, - jupyter_cli, - pipeline_cli, - micropkg_cli, - project_group, - registry_cli, + # catalog_cli, + # jupyter_cli, + # pipeline_cli, + # micropkg_cli, + # project_group, + # registry_cli, cli, + global_commands, ] def test_kedro_cli_no_project(self, mocker, tmp_path): mocker.patch("kedro.framework.cli.cli._is_project", return_value=False) kedro_cli = KedroCLI(tmp_path) assert len(kedro_cli.global_groups) == 2 - assert kedro_cli.global_groups == [cli, create_cli] + assert kedro_cli.global_groups == [cli, global_commands] result = CliRunner().invoke(kedro_cli, []) @@ -422,16 +415,17 @@ def test_kedro_cli_with_project(self, mocker, fake_metadata): kedro_cli = KedroCLI(fake_metadata.project_path) assert len(kedro_cli.global_groups) == 2 - assert kedro_cli.global_groups == [cli, create_cli] - assert len(kedro_cli.project_groups) == 7 + assert kedro_cli.global_groups == [cli, global_commands] + assert len(kedro_cli.project_groups) == 2 assert kedro_cli.project_groups == [ - catalog_cli, - jupyter_cli, - pipeline_cli, - micropkg_cli, - project_group, - registry_cli, + # catalog_cli, + # jupyter_cli, + # pipeline_cli, + # micropkg_cli, + # project_group, + # registry_cli, cli, + project_commands, ] result = CliRunner().invoke(kedro_cli, []) From 9af13dfd2f142e21c817da38c7bcad718b739c93 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Fri, 28 Jun 2024 16:58:09 +0100 Subject: [PATCH 09/17] Some tests + lint Signed-off-by: Ankita Katiyar --- kedro/framework/cli/lazy_group.py | 12 +++++++++--- tests/framework/cli/test_cli.py | 16 ++-------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py index b28d901666..eb197f7066 100644 --- a/kedro/framework/cli/lazy_group.py +++ b/kedro/framework/cli/lazy_group.py @@ -1,12 +1,18 @@ from __future__ import annotations import importlib +from typing import Any import click class LazyGroup(click.Group): - def __init__(self, *args, lazy_subcommands: dict[str, str] | None = None, **kwargs): + def __init__( + self, + *args: Any, + lazy_subcommands: dict[str, str] | None = None, + **kwargs: Any, + ): super().__init__(*args, **kwargs) # lazy_subcommands is a map of the form: # @@ -19,9 +25,9 @@ def list_commands(self, ctx: click.Context) -> list[str]: lazy = sorted(self.lazy_subcommands.keys()) return base + lazy - def get_command( + def get_command( # type: ignore[override] self, ctx: click.Context, cmd_name: str - ) -> click.BaseCommand | click.Command | None: # type: ignore[override] + ) -> click.BaseCommand | click.Command | None: if cmd_name in self.lazy_subcommands: return self._lazy_load(cmd_name) return super().get_command(ctx, cmd_name) diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index 5eaa6e643e..e910e59eea 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -372,7 +372,7 @@ def test_project_commands_valid_clipy(self, mocker, fake_metadata): # project_group, # registry_cli, cli, - global_commands, + project_commands, ] def test_kedro_cli_no_project(self, mocker, tmp_path): @@ -403,28 +403,16 @@ def test_kedro_run_no_project(self, mocker, tmp_path): ) def test_kedro_cli_with_project(self, mocker, fake_metadata): - Module = namedtuple("Module", ["cli"]) mocker.patch("kedro.framework.cli.cli._is_project", return_value=True) mocker.patch( "kedro.framework.cli.cli.bootstrap_project", return_value=fake_metadata ) - mocker.patch( - "kedro.framework.cli.cli.importlib.import_module", - return_value=Module(cli=cli), - ) kedro_cli = KedroCLI(fake_metadata.project_path) assert len(kedro_cli.global_groups) == 2 assert kedro_cli.global_groups == [cli, global_commands] - assert len(kedro_cli.project_groups) == 2 + assert len(kedro_cli.project_groups) == 1 assert kedro_cli.project_groups == [ - # catalog_cli, - # jupyter_cli, - # pipeline_cli, - # micropkg_cli, - # project_group, - # registry_cli, - cli, project_commands, ] From 7933c61c1c915cda190b7087bf107c3f1cadccf2 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 1 Jul 2024 11:45:23 +0100 Subject: [PATCH 10/17] fix tests Signed-off-by: Ankita Katiyar --- tests/framework/cli/test_cli_hooks.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/framework/cli/test_cli_hooks.py b/tests/framework/cli/test_cli_hooks.py index 57f8039e9f..7985de52fd 100644 --- a/tests/framework/cli/test_cli_hooks.py +++ b/tests/framework/cli/test_cli_hooks.py @@ -6,7 +6,7 @@ import pytest from click.testing import CliRunner -from kedro.framework.cli.cli import KedroCLI, cli +from kedro.framework.cli.cli import KedroCLI from kedro.framework.cli.hooks import cli_hook_impl, get_cli_hook_manager, manager from kedro.framework.startup import ProjectMetadata @@ -83,7 +83,7 @@ def fake_plugin_distribution(mocker): class TestKedroCLIHooks: @pytest.mark.parametrize( "command, exit_code", - [("-V", 0), ("info", 2), ("pipeline list", 2), ("starter", 0)], + [("-V", 0), ("info", 0), ("pipeline list", 2), ("starter", 0)], ) def test_kedro_cli_should_invoke_cli_hooks_from_plugin( self, @@ -97,7 +97,7 @@ def test_kedro_cli_should_invoke_cli_hooks_from_plugin( ): caplog.set_level(logging.DEBUG, logger="kedro") - Module = namedtuple("Module", ["cli"]) + # Module = namedtuple("Module", ["cli"]) mocker.patch( "kedro.framework.cli.cli._is_project", return_value=True, @@ -106,10 +106,10 @@ def test_kedro_cli_should_invoke_cli_hooks_from_plugin( "kedro.framework.cli.cli.bootstrap_project", return_value=fake_metadata, ) - mocker.patch( - "kedro.framework.cli.cli.importlib.import_module", - return_value=Module(cli=cli), - ) + # mocker.patch( + # "kedro.framework.cli.cli.importlib.import_module", + # return_value=Module(cli=cli), + # ) kedro_cli = KedroCLI(fake_metadata.project_path) result = CliRunner().invoke(kedro_cli, [command]) assert ( @@ -121,8 +121,7 @@ def test_kedro_cli_should_invoke_cli_hooks_from_plugin( f"Before command `{command}` run for project {fake_metadata}" in result.output ) - - # 'pipeline list' and 'info' aren't actually in the click structure and + # 'pipeline list' isn't actually in the click structure and # return exit code 2 ('invalid usage of some shell built-in command') assert ( f"After command `{command}` run for project {fake_metadata} (exit: {exit_code})" From 409621e38d099216e1b5ccc49e39db30dfc8a52e Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Jul 2024 14:01:33 +0100 Subject: [PATCH 11/17] Move lazy group Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 6 ++-- kedro/framework/cli/lazy_group.py | 49 ------------------------------- kedro/framework/cli/utils.py | 37 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 52 deletions(-) delete mode 100644 kedro/framework/cli/lazy_group.py diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index a7bdc0bf2e..ddd8fa4cb4 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -16,12 +16,12 @@ from kedro import __version__ as version from kedro.framework.cli import BRIGHT_BLACK, ORANGE from kedro.framework.cli.hooks import get_cli_hook_manager -from kedro.framework.cli.lazy_group import LazyGroup from kedro.framework.cli.utils import ( CONTEXT_SETTINGS, ENTRY_POINT_GROUPS, CommandCollection, KedroCliError, + LazyGroup, _get_entry_points, load_entry_points, ) @@ -95,7 +95,7 @@ def info() -> None: }, ) def project_commands() -> None: - pass + pass # pragma: no cover @click.group( @@ -108,7 +108,7 @@ def project_commands() -> None: }, ) def global_commands() -> None: - pass + pass # pragma: no cover def _init_plugins() -> None: diff --git a/kedro/framework/cli/lazy_group.py b/kedro/framework/cli/lazy_group.py deleted file mode 100644 index eb197f7066..0000000000 --- a/kedro/framework/cli/lazy_group.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -import importlib -from typing import Any - -import click - - -class LazyGroup(click.Group): - def __init__( - self, - *args: Any, - lazy_subcommands: dict[str, str] | None = None, - **kwargs: Any, - ): - super().__init__(*args, **kwargs) - # lazy_subcommands is a map of the form: - # - # {command-name} -> {module-name}.{command-object-name} - # - self.lazy_subcommands = lazy_subcommands or {} - - def list_commands(self, ctx: click.Context) -> list[str]: - base = list(super().list_commands(ctx)) - lazy = sorted(self.lazy_subcommands.keys()) - return base + lazy - - def get_command( # type: ignore[override] - self, ctx: click.Context, cmd_name: str - ) -> click.BaseCommand | click.Command | None: - if cmd_name in self.lazy_subcommands: - return self._lazy_load(cmd_name) - return super().get_command(ctx, cmd_name) - - def _lazy_load(self, cmd_name: str) -> click.BaseCommand: - # lazily loading a command, first get the module name and attribute name - import_path = self.lazy_subcommands[cmd_name] - modname, cmd_object_name = import_path.rsplit(".", 1) - # do the import - mod = importlib.import_module(modname) - # get the Command object from that module - cmd_object = getattr(mod, cmd_object_name) - # check the result to make debugging easier - if not isinstance(cmd_object, click.BaseCommand): - raise ValueError( - f"Lazy loading of {import_path} failed by returning " - "a non-command object" - ) - return cmd_object diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index dde485189b..7cca44b521 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -502,3 +502,40 @@ def _split_load_versions(ctx: click.Context, param: Any, value: str) -> dict[str load_versions_dict[load_version_list[0]] = load_version_list[1] return load_versions_dict + + +class LazyGroup(click.Group): + def __init__( + self, + *args: Any, + lazy_subcommands: dict[str, str] | None = None, + **kwargs: Any, + ): + super().__init__(*args, **kwargs) + # lazy_subcommands is a map of the form: + # + # {command-name} -> {module-name}.{command-object-name} + # + self.lazy_subcommands = lazy_subcommands or {} + + def list_commands(self, ctx: click.Context) -> list[str]: + base = list(super().list_commands(ctx)) + lazy = sorted(self.lazy_subcommands.keys()) + return base + lazy + + def get_command( # type: ignore[override] + self, ctx: click.Context, cmd_name: str + ) -> click.BaseCommand | click.Command | None: + if cmd_name in self.lazy_subcommands: + return self._lazy_load(cmd_name) + return super().get_command(ctx, cmd_name) + + def _lazy_load(self, cmd_name: str) -> click.BaseCommand: + # lazily loading a command, first get the module name and attribute name + import_path = self.lazy_subcommands[cmd_name] + modname, cmd_object_name = import_path.rsplit(".", 1) + # do the import + mod = importlib.import_module(modname) + # get the Command object from that module + cmd_object = getattr(mod, cmd_object_name) + return cmd_object From dfba836bc373558bd4557e00aefb5a23ee841dec Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Jul 2024 14:09:43 +0100 Subject: [PATCH 12/17] lint Signed-off-by: Ankita Katiyar --- kedro/framework/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index 7cca44b521..052efd6754 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -535,7 +535,7 @@ def _lazy_load(self, cmd_name: str) -> click.BaseCommand: import_path = self.lazy_subcommands[cmd_name] modname, cmd_object_name = import_path.rsplit(".", 1) # do the import - mod = importlib.import_module(modname) + mod = import_module(modname) # get the Command object from that module cmd_object = getattr(mod, cmd_object_name) return cmd_object From 4adb0b86ef981e559dc274199da74a372c17b4d7 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Jul 2024 15:53:35 +0100 Subject: [PATCH 13/17] lint Signed-off-by: Ankita Katiyar --- kedro/framework/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index 052efd6754..3f955a407d 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -537,5 +537,5 @@ def _lazy_load(self, cmd_name: str) -> click.BaseCommand: # do the import mod = import_module(modname) # get the Command object from that module - cmd_object = getattr(mod, cmd_object_name) + cmd_object = getattr(mod, cmd_object_name) # type: ignore[no-any-return] return cmd_object From 8a903bf0ea4675b75edbba1711326aeb74316857 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 8 Jul 2024 16:02:06 +0100 Subject: [PATCH 14/17] lint Signed-off-by: Ankita Katiyar --- kedro/framework/cli/cli.py | 2 +- kedro/framework/cli/utils.py | 30 ++++-------------------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index ddd8fa4cb4..40e5509d07 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -203,7 +203,7 @@ def global_groups(self) -> Sequence[click.MultiCommand]: combines them with the built-in ones (eventually overriding the built-in ones if they are redefined by plugins). """ - return [cli, global_commands, *load_entry_points("global")] + return [cli, *load_entry_points("global"), global_commands] @property def project_groups(self) -> Sequence[click.MultiCommand]: diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py index 3f955a407d..7258fb2680 100644 --- a/kedro/framework/cli/utils.py +++ b/kedro/framework/cli/utils.py @@ -126,7 +126,6 @@ def __init__(self, *groups: tuple[str, Sequence[click.MultiCommand]]): for cli in cli_collection.sources if cli.help ] - # self._dedupe_commands(sources) super().__init__( sources=sources, # type: ignore[arg-type] help="\n\n".join(help_texts), @@ -135,29 +134,6 @@ def __init__(self, *groups: tuple[str, Sequence[click.MultiCommand]]): self.params = sources[0].params self.callback = sources[0].callback - # @staticmethod - # def _dedupe_commands(cli_collections: Sequence[click.CommandCollection]) -> None: - # """Deduplicate commands by keeping the ones from the last source - # in the list. - # """ - # seen_names: set[str] = set() - # for cli_collection in reversed(cli_collections): - # for cmd_group in reversed(cli_collection.sources): - # cmd_group.commands = { # type: ignore[attr-defined] - # cmd_name: cmd - # for cmd_name, cmd in cmd_group.commands.items() # type: ignore[attr-defined] - # if cmd_name not in seen_names - # } - # seen_names |= cmd_group.commands.keys() # type: ignore[attr-defined] - - # # remove empty command groups - # for cli_collection in cli_collections: - # cli_collection.sources = [ - # cmd_group - # for cmd_group in cli_collection.sources - # if cmd_group.commands # type: ignore[attr-defined] - # ] - @staticmethod def _merge_same_name_collections( groups: Sequence[click.MultiCommand], @@ -505,6 +481,8 @@ def _split_load_versions(ctx: click.Context, param: Any, value: str) -> dict[str class LazyGroup(click.Group): + """A click Group that supports lazy loading of subcommands.""" + def __init__( self, *args: Any, @@ -537,5 +515,5 @@ def _lazy_load(self, cmd_name: str) -> click.BaseCommand: # do the import mod = import_module(modname) # get the Command object from that module - cmd_object = getattr(mod, cmd_object_name) # type: ignore[no-any-return] - return cmd_object + cmd_object = getattr(mod, cmd_object_name) + return cmd_object # type: ignore[no-any-return] From 0fd6cb4c50415164333adff3711a1d662c56de5c Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Tue, 9 Jul 2024 15:20:01 +0100 Subject: [PATCH 15/17] Remove commented parts, add comments Signed-off-by: Ankita Katiyar --- docs/source/kedro_project_setup/session.md | 2 +- kedro/framework/cli/cli.py | 5 ++++- tests/framework/cli/test_cli.py | 11 ++++------- tests/framework/cli/test_cli_hooks.py | 5 ----- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/docs/source/kedro_project_setup/session.md b/docs/source/kedro_project_setup/session.md index e1a95bf668..3b736db641 100644 --- a/docs/source/kedro_project_setup/session.md +++ b/docs/source/kedro_project_setup/session.md @@ -67,4 +67,4 @@ This function reads `settings.py` and `pipeline_registry.py` and registers the c #### ValueError: Package name not found > ValueError: Package name not found. Make sure you have configured the project using 'bootstrap_project'. This should happen automatically if you are using Kedro command line interface. -If you are using `multiprocessing`, you need to be careful about this. Depending on your Operating System, you may have [different default](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods). If the processes are `spawn`, Python will re-import all the modules in each process and thus you need to run `configure_project` again at the start of the new process. For example, this is how Kedro handle this in `ParallelRunner`(https://github.com/kedro-org/kedro/blob/9e883e6a0ba40e3db4497b234dcb3801258e8396/kedro/runner/parallel_runner.py#L84-L85) +If you are using `multiprocessing`, you need to be careful about this. Depending on your Operating System, you may have [different default](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods). If the processes are `spawn`, Python will re-import all the modules in each process and thus you need to run `configure_project` again at the start of the new process. For example, [this is how Kedro handles this in `ParallelRunner`](https://github.com/kedro-org/kedro/blob/9e883e6a0ba40e3db4497b234dcb3801258e8396/kedro/runner/parallel_runner.py#L84-L85) diff --git a/kedro/framework/cli/cli.py b/kedro/framework/cli/cli.py index 40e5509d07..f90f7ab160 100644 --- a/kedro/framework/cli/cli.py +++ b/kedro/framework/cli/cli.py @@ -45,6 +45,9 @@ def cli() -> None: # pragma: no cover """Kedro is a CLI for creating and using Kedro projects. For more information, type ``kedro info``. + NOTE: If a command from a plugin conflicts with a built-in command from Kedro, + the command from the plugin will take precedence. + """ pass @@ -203,7 +206,7 @@ def global_groups(self) -> Sequence[click.MultiCommand]: combines them with the built-in ones (eventually overriding the built-in ones if they are redefined by plugins). """ - return [cli, *load_entry_points("global"), global_commands] + return [*load_entry_points("global"), cli, global_commands] @property def project_groups(self) -> Sequence[click.MultiCommand]: diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index e910e59eea..818cd54d67 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -331,7 +331,7 @@ def test_project_commands_no_clipy(self, mocker, fake_metadata): side_effect=cycle([ModuleNotFoundError()]), ) kedro_cli = KedroCLI(fake_metadata.project_path) - print(kedro_cli.project_groups) + # There is only one `LazyGroup` for project commands assert len(kedro_cli.project_groups) == 1 assert kedro_cli.project_groups == [project_commands] @@ -363,14 +363,10 @@ def test_project_commands_valid_clipy(self, mocker, fake_metadata): return_value=Module(cli=cli), ) kedro_cli = KedroCLI(fake_metadata.project_path) + # The project group will now have two groups, the first from the project's cli.py and + # the second is the lazy project command group assert len(kedro_cli.project_groups) == 2 assert kedro_cli.project_groups == [ - # catalog_cli, - # jupyter_cli, - # pipeline_cli, - # micropkg_cli, - # project_group, - # registry_cli, cli, project_commands, ] @@ -379,6 +375,7 @@ def test_kedro_cli_no_project(self, mocker, tmp_path): mocker.patch("kedro.framework.cli.cli._is_project", return_value=False) kedro_cli = KedroCLI(tmp_path) assert len(kedro_cli.global_groups) == 2 + # The global groups will be the cli(group for info command) and the global commands (starter and new) assert kedro_cli.global_groups == [cli, global_commands] result = CliRunner().invoke(kedro_cli, []) diff --git a/tests/framework/cli/test_cli_hooks.py b/tests/framework/cli/test_cli_hooks.py index 7985de52fd..33f13e50fb 100644 --- a/tests/framework/cli/test_cli_hooks.py +++ b/tests/framework/cli/test_cli_hooks.py @@ -97,7 +97,6 @@ def test_kedro_cli_should_invoke_cli_hooks_from_plugin( ): caplog.set_level(logging.DEBUG, logger="kedro") - # Module = namedtuple("Module", ["cli"]) mocker.patch( "kedro.framework.cli.cli._is_project", return_value=True, @@ -106,10 +105,6 @@ def test_kedro_cli_should_invoke_cli_hooks_from_plugin( "kedro.framework.cli.cli.bootstrap_project", return_value=fake_metadata, ) - # mocker.patch( - # "kedro.framework.cli.cli.importlib.import_module", - # return_value=Module(cli=cli), - # ) kedro_cli = KedroCLI(fake_metadata.project_path) result = CliRunner().invoke(kedro_cli, [command]) assert ( From 37764a7a25698835341e1d9d64ee5a8f73971ec0 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Tue, 9 Jul 2024 15:36:00 +0100 Subject: [PATCH 16/17] test lazy commands + docs link Signed-off-by: Ankita Katiyar --- docs/source/kedro_project_setup/session.md | 6 +++++- tests/framework/cli/test_cli.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/source/kedro_project_setup/session.md b/docs/source/kedro_project_setup/session.md index 3b736db641..a862a36447 100644 --- a/docs/source/kedro_project_setup/session.md +++ b/docs/source/kedro_project_setup/session.md @@ -67,4 +67,8 @@ This function reads `settings.py` and `pipeline_registry.py` and registers the c #### ValueError: Package name not found > ValueError: Package name not found. Make sure you have configured the project using 'bootstrap_project'. This should happen automatically if you are using Kedro command line interface. -If you are using `multiprocessing`, you need to be careful about this. Depending on your Operating System, you may have [different default](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods). If the processes are `spawn`, Python will re-import all the modules in each process and thus you need to run `configure_project` again at the start of the new process. For example, [this is how Kedro handles this in `ParallelRunner`](https://github.com/kedro-org/kedro/blob/9e883e6a0ba40e3db4497b234dcb3801258e8396/kedro/runner/parallel_runner.py#L84-L85) +If you are using `multiprocessing`, you need to be careful about this. Depending on your Operating System, you may have [different default](https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods). If the processes are `spawn`, Python will re-import all the modules in each process and thus you need to run `configure_project` again at the start of the new process. For example, this is how Kedro handles this in `ParallelRunner`: +```python +if multiprocessing.get_start_method() == "spawn" and package_name: + _bootstrap_subprocess(package_name, logging_config) +``` diff --git a/tests/framework/cli/test_cli.py b/tests/framework/cli/test_cli.py index 818cd54d67..ffd64cd639 100644 --- a/tests/framework/cli/test_cli.py +++ b/tests/framework/cli/test_cli.py @@ -334,6 +334,17 @@ def test_project_commands_no_clipy(self, mocker, fake_metadata): # There is only one `LazyGroup` for project commands assert len(kedro_cli.project_groups) == 1 assert kedro_cli.project_groups == [project_commands] + # Assert that the lazy commands are listed properly + assert kedro_cli.project_groups[0].list_commands(None) == [ + "catalog", + "ipython", + "jupyter", + "micropkg", + "package", + "pipeline", + "registry", + "run", + ] def test_project_commands_no_project(self, mocker, tmp_path): mocker.patch("kedro.framework.cli.cli._is_project", return_value=False) From d78a4c6f00692752d31f7bb176be47d416c93854 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Tue, 9 Jul 2024 15:57:51 +0100 Subject: [PATCH 17/17] Release notes Signed-off-by: Ankita Katiyar --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 04de728c30..6d642014e7 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,7 @@ # Upcoming Release 0.19.7 ## Major features and improvements +* Kedro commands are now lazily loaded to add performance gains when running Kedro commands. ## Bug fixes and other changes * Updated error message for invalid catalog entries.