feat: cli tester

cliffy/builder.py

@@ -1,18 +1,16 @@
 from io import TextIOWrapper
 import os
 import sys
-from importlib import import_module
 from pathlib import Path
 from shutil import copy
 from tempfile import NamedTemporaryFile, TemporaryDirectory
-from types import ModuleType
 from typing import Optional
 
 from click.testing import CliRunner, Result
 from shiv import cli as shiv_cli
 from shiv import pip
 
-from cliffy.helper import TEMP_FILES, delete_temp_files
+from cliffy.helper import TEMP_FILES, delete_temp_files, import_module_from_path
 
 from cliffy.transformer import Transformer
 
@@ -62,12 +60,6 @@ def build_cli(
         )
 
 
-def import_module_from_path(filepath: str) -> ModuleType:
-    module_path, module_filename = os.path.split(filepath)
-    sys.path.append(module_path)
-    return import_module(module_filename[:-3])
-
-
 def run_cli(cli_name: str, script_code: str, args: tuple) -> None:
     with NamedTemporaryFile(mode="w", prefix=f"{cli_name}_", suffix=".py", delete=False) as runner_file:
         runner_file.write(script_code)

cliffy/cli.py

@@ -2,10 +2,13 @@
 from io import TextIOWrapper
 import os
 from typing import IO, Any, Optional, TextIO, Union, cast
-
+import traceback
+import sys
 from click.core import Context, Parameter
 from click.types import _is_file_like
 
+from cliffy.tester import Tester
+
 from .rich import click, Console, print_rich_table
 
 from .builder import build_cli, build_cli_from_manifest, run_cli
@@ -22,7 +25,7 @@
 from .loader import Loader
 from .manifests import Manifest, set_manifest_version
 from .transformer import Transformer
-from .reloader import CLIManifestReloader
+from .reloader import Reloader
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 ALIASES = {
@@ -236,7 +239,36 @@ def dev(manifest: str, run_cli: bool, run_cli_args: tuple[str]) -> None:
     - cli dev examples/hello.yaml --run-cli hello
     """
     out(f"Watching {manifest} for changes\n", fg="magenta")
-    CLIManifestReloader.watch(manifest, run_cli, run_cli_args)
+    Reloader.watch(manifest, run_cli, run_cli_args)
+
+
+@click.argument("manifest", type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True))
+def test(manifest: str) -> None:
+    """Run tests defined in a manifest"""
+    tester = Tester(manifest)
+    out("✨ Invoking tests ✨", nl=False)
+    total = len(tester.T.manifest.tests)
+    for i, (command, script) in enumerate(tester.T.manifest.tests.items()):
+        out(f"\n\n🪄 > {tester.T.cli.name} {command}\n")
+        try:
+            test = tester.invoke_test(command, script)
+            result = next(test)
+            if result.exception:
+                out(str(result))
+            next(test, "")
+            out(f"\n💚 {i+1}/{total}")
+        except AssertionError:
+            _, _, tb = sys.exc_info()
+            tb_info = traceback.extract_tb(tb)
+            _, line_no, _, _ = tb_info[-1]
+            expr = script.split("\n")[line_no - 1]
+            out(f"💔 AssertionError: (line {line_no}) > {expr}")
+        except SyntaxError:
+            out("💔 Syntax error")
+            traceback.print_exc()
+        except Exception:
+            out("💔 Exception")
+            traceback.print_exc()
 
 
 # register commands
@@ -251,6 +283,7 @@ def dev(manifest: str, run_cli: bool, run_cli_args: tuple[str]) -> None:
 remove_all_command = cli.command("remove-all")(remove_all)
 run_command = cli.command("run")(cliffy_run)
 update_command = cli.command("update")(update)
+test_command = cli.command("test")(test)
 
 # register aliases
 cli.command("add", hidden=True, epilog="Alias for load")(load)

cliffy/helper.py

@@ -13,6 +13,8 @@
 from click import secho
 from packaging import version
 from pydantic import BaseModel
+from types import ModuleType
+from importlib import import_module
 
 
 CLIFFY_CLI_DIR = files("cliffy").joinpath("clis")
@@ -49,6 +51,12 @@ def write_to_file(path: str, text: str, executable: bool = False) -> None:
         make_executable(path)
 
 
+def import_module_from_path(filepath: str) -> ModuleType:
+    module_path, module_filename = os.path.split(filepath)
+    sys.path.append(module_path)
+    return import_module(module_filename[:-3])
+
+
 def make_executable(path: str) -> None:
     mode = os.stat(path).st_mode
     mode |= (mode & 0o444) >> 2

cliffy/manifests/v1.py

@@ -77,6 +77,7 @@ class CLIManifest(BaseModel):
         {},
         description="A mapping for any additional options that can be used to customize the behavior of the CLI.",
     )
+    tests: dict[str, str] = Field({}, description="A mapping of command with args to its test script.")
 
     @classmethod
     def get_field_description(cls, field_name: str, as_comment: bool = True) -> str:
@@ -208,6 +209,7 @@ class IncludeManifest(BaseModel):
     functions: list[str] = []
     types: dict[str, str] = {}
     cli_options: dict[str, str] = {}
+    tests: dict[str, str] = {}
 
 
 class CLIMetadata(BaseModel):

cliffy/reloader.py

@@ -12,7 +12,7 @@
 import threading
 
 
-class CLIManifestReloader(FileSystemEventHandler):
+class Reloader(FileSystemEventHandler):
     def __init__(self, manifest_path: str, run_cli: bool, run_cli_args: tuple[str]) -> None:
         self.manifest_path = manifest_path
         self.run_cli = run_cli

cliffy/tester.py

@@ -0,0 +1,40 @@
+from typing import Generator
+from click.testing import Result
+from typer.testing import CliRunner
+from cliffy.commander import Command
+from cliffy.parser import Parser
+from cliffy.transformer import Transformer
+from cliffy.helper import import_module_from_path, delete_temp_files, TEMP_FILES
+from tempfile import NamedTemporaryFile
+import inspect
+
+
+class Tester:
+    def __init__(self, manifest_path: str) -> None:
+        with open(manifest_path, "r") as manifest_io:
+            self.T = Transformer(manifest_io)
+
+        with NamedTemporaryFile(mode="w", prefix=f"{self.T.cli.name}_test_", suffix=".py", delete=False) as runner_file:
+            runner_file.write(self.T.cli.code)
+            runner_file.flush()
+            self.module = import_module_from_path(runner_file.name)
+            TEMP_FILES.append(runner_file)
+
+        self.module_funcs = inspect.getmembers(self.module, inspect.isfunction)
+        delete_temp_files()
+
+        self.runner = CliRunner()
+        self.parser = Parser(self.T.manifest)
+
+    def invoke_test(self, command: str, script: str) -> Generator[Result, None, None]:
+        result = self.runner.invoke(self.module.cli, command)
+        yield result
+        code = compile(script, f"test_{self.T.cli.name}.py", "exec")
+        exec(code, {}, {"result": result, "result_text": result.output.strip()})
+
+    def is_valid_command(self, command: str) -> bool:
+        command_name = command.split(" ")[0]
+        cmd = Command(name=command_name, script="")
+        command_func_name = self.parser.get_command_func_name(cmd)
+        matching_command_func = [func for fname, func in self.module_funcs if fname == command_func_name]
+        return bool(matching_command_func)

cliffy/transformer.py

@@ -18,7 +18,14 @@ class Transformer:
 
     __slots__ = ("manifest_io", "command_config", "manifest_version", "includes_config", "manifest", "cli")
 
-    def __init__(self, manifest_io: TextIO, *, as_include: bool = False, validate_requires: bool = True) -> None:
+    def __init__(
+        self,
+        manifest_io: TextIO,
+        *,
+        as_include: bool = False,
+        validate_requires: bool = True,
+        add_command_name_mapping: bool = False,
+    ) -> None:
         self.manifest_io = manifest_io
         self.command_config = self.load_manifest(manifest_io)
         self.manifest_version = self.command_config.pop("manifestVersion", "v1")

examples/generated/db.py

@@ -1,4 +1,4 @@
-## Generated db on 2024-08-07 10:34:35.556438
+## Generated db on 2024-08-25 23:31:48.012819
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/environ.py

@@ -1,4 +1,4 @@
-## Generated environ on 2024-08-07 10:34:35.562029
+## Generated environ on 2024-08-25 23:31:48.018345
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/hello.py

@@ -1,4 +1,4 @@
-## Generated hello on 2024-08-07 10:34:35.566032
+## Generated hello on 2024-08-25 23:31:48.022732
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/penv.py

@@ -1,4 +1,4 @@
-## Generated penv on 2024-08-07 10:34:35.569851
+## Generated penv on 2024-08-25 23:31:48.026516
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/pydev.py

@@ -1,4 +1,4 @@
-## Generated pydev on 2024-08-07 10:34:35.574671
+## Generated pydev on 2024-08-25 23:31:48.031791
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/requires.py

@@ -1,4 +1,4 @@
-## Generated requires on 2024-08-07 10:34:35.763264
+## Generated requires on 2024-08-25 23:31:48.327128
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/template.py

@@ -1,4 +1,4 @@
-## Generated template on 2024-08-07 10:34:35.766497
+## Generated template on 2024-08-25 23:31:48.330347
 import typer
 import subprocess
 from typing import Optional, Any

examples/generated/town.py

@@ -1,4 +1,4 @@
-## Generated town on 2024-08-07 10:34:35.773337
+## Generated town on 2024-08-25 23:31:48.337317
 import typer
 import subprocess
 from typing import Optional, Any

examples/hello.yaml

@@ -4,4 +4,8 @@ help: Hello world!
 
 commands:
   bash: $echo "hello from bash"
-  python: print("hello from python")
+  python: print("hello from python")
+
+tests:
+  bash: assert result.exit_code == 0
+  python: assert result.output == "hello from python\n"