From 49e4765c407ba5f3813ce4c2639e299c3555e123 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Thu, 15 Aug 2024 11:18:15 +0100 Subject: [PATCH] handle dashed namespace names --- README.md | 142 ++++++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- una/check.py | 5 +- una/cli.py | 11 ++-- una/config.py | 12 ++-- una/defaults.py | 1 + una/differ.py | 2 +- una/external_deps.py | 8 ++- una/files.py | 11 ++-- una/internal_deps.py | 3 +- una/sync.py | 3 +- 11 files changed, 159 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 694890a..97c702e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,125 @@ # una **Warning: this is pre-alpha and probably doesn't work at all. You'll probably just get frustrated if you even try to use it.** -una is a tool to make Python monorepos with Hatch and/or Rye easier. It is a CLI tool and a Hatch plugin that does the following things: +una is a tool to make Python monorepos with Rye easier. It is a CLI tool and a Hatch plugin that does the following things: 1. Enable builds of individual apps or projects within a monorepo. 2. Ensure that internal and external dependencies are correctly specified. -una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and is based on that codebase. But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. It just tries to make a simple monorepo build just about possible. +una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and is based on that codebase. +But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. +It just tries to make a simple monorepo build just about possible. una allows two directory structures or styles: - `packages`: this is the lightest-touch approach, that is just some extra build help on top of a Rye workspace. -- modules: a more novel approach with just a single pyproject.toml, arguably better DevX and compatible with Rye or Hatch alone. +- `modules`: a more novel approach with just a single pyproject.toml, arguably better DevX and doesn't require a Rye workspace. Within this context, we use the following words frequently: - `lib`: a module or package that will be imported but not run. - `app`: a module or package that will be run but never imported. - `project`: a package with no code but only dependencies (only used in the `modules` style) +## Quickstart +This will give you a quick view of how this all works. +A `packages` style will be used by default, as it is probably more familiar to most. + +```bash +rye init my_example +cd my_example +rye add --dev una + +# this will create a bunch of folders and files, have a look at them! +rye run una create workspace + +# why not have a look +tree +``` + +The let's create some internal and external dependencies: +```bash +# add an external library to example_lib +cd libs/example_lib +rye add urllib3 +cd ../.. + +# and then depend on example_lib from example_app +echo "import my_example.example_lib" > apps/example_app/my_example/example_app/foo.py +``` + +But then how do we ensure that builds of example_app will include all of the required code and dependencies? +```bash +# this checks all imports and ensures they are added to +# [tool.una.libs] in the appropriate pyproject.toml +rye run una sync + +# have a look at what happened +tail apps/example_app/pyproject.toml +``` + +Now you can build your app: +```bash +rye build --package example_app + +# and see the result +ls dist/ +``` + +And you can do whatever you want with that wheel! +What about stick it in a Dockerfile, have you ever seen such a simple one? +```Dockerfile +FROM python +COPY dist dist +RUN pip install dist/*.whl +``` + +## Installation +The CLI tool isn't strictly necessary, as all the stuff that lets the monorepo builds work is in the separate (and tiny) [hatch-una](plugins/hatch) package. +But you will likely struggle to manage your monorepo without the tool! + +So you may as well install it: +```bash +pip install una +``` + +As for the build-time `hatch-una`, it will automatically be installed by build tools when it spots this in your `pyproject.toml` (this will be configured automatically by the CLI): +```toml +[build-system] +requires = ["hatchling", "hatch-una"] +build-backend = "hatchling.build" +``` + +## Usage +The CLI has a few commands and options, have a look: +```bash +rye run una --help + + Usage: una [OPTIONS] COMMAND [ARGS]... + +╭─ Options ───────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰─────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ──────────────────────────────────────────────────────────────╮ +│ create Commands for creating a workspace, apps, libs and projects. │ +│ diff Shows changed int_deps compared to the latest git tag. │ +│ info Info about the Una workspace. │ +│ sync Update pyproject.toml with missing int_deps. │ +╰─────────────────────────────────────────────────────────────────────────╯ +``` + +## Pyright +With the `packages` style (the default), you'll need do Pyright type-checking on a per-package basis. + +That is, you should add something like the config below to each `apps/*/pyproject.toml` and `libs/*/pyproject.toml`. + +With the `modules` style this is not necessary, and you can just have one root Pyright (and Pytest!). + +```toml +[tool.pyright] +venvPath = "../.." +venv = ".venv" +pythonVersion = "3.11" +strict = ["**/*.py"] +``` + ## Style: Packages In this setup, we use Rye's built-in workspace support. The structure will look something like this: ```bash @@ -44,7 +148,8 @@ In this setup, we use Rye's built-in workspace support. The structure will look This means: 1. Each `app` or `lib` (collectively, internal dependencies) is it's own Python package with a `pyproject.toml`. 2. You must specify the workspace members in `tool.rye.workspace.members`. -3. Type-checking and testing should be done on a per-package level. That is, you should run `pyright` and `pytest` from `apps/server` or `libs/mylib`, _not_ from the root. +3. Type-checking and testing should be done on a per-package level. +That is, you should run `pyright` and `pytest` from `apps/server` or `libs/mylib`, _not_ from the root. In the example above, the only build artifact will be for `apps/server`. At build-time, una will do the following: 1. Read the list of internal dependencies (more on this shortly) and inject them into the build. @@ -62,8 +167,10 @@ virtual = true members = ["apps/*", "libs/*"] ``` -2. Create your apps and your libs as you would, ensuring that app code is never imported. Ensure that you choose a good namespace and always use it in your package structures (check `your_ns` in the example structure above.) -3. Add external dependencies to your libs and apps as normal. Then, to add an internal dependency to an app, we do the following in its pyproject.toml: +2. Create your apps and your libs as you would, ensuring that app code is never imported. +Ensure that you choose a good namespace and always use it in your package structures (check `your_ns` in the example structure above.) +3. Add external dependencies to your libs and apps as normal. +Then, to add an internal dependency to an app, we do the following in its pyproject.toml: ```toml # /apps/server/pyproject.toml [build-system] @@ -75,19 +182,17 @@ build-backend = "hatchling.build" [tool.una.libs] "../../libs/mylib/example/mylib" = "example/mylib" ``` -4. Then you can run `rye build --wheel` from that package directory and una will inject everything that is needed. Once you have your built `.whl` file, all you need in your Dockerfile is: +4. Then you can run `rye build --wheel` from that package directory and una will inject everything that is needed. +Once you have your built `.whl` file, all you need in your Dockerfile is: ```Dockerfile FROM python -COPY dist . +COPY dist dist RUN pip install dist/*.whl ``` -### Historical note -What is needed to get root-level pyright and pytest to work? -1. Delete apps/lib pyproject.toml as they cause pyright/basedpyright to get lost. -2. Add pythonpath to pytest settings. ## Style: Modules -This approach is inspired by [Polylith](https://davidvujic.github.io/python-polylith-docs/). You don't use a Rye workspace (and indeed this approach will work with just Hatch), and there's only a single `pyproject.toml`. +This approach is inspired by [Polylith](https://davidvujic.github.io/python-polylith-docs/). +You don't use a Rye workspace (and indeed this approach will work with just Hatch), and there's only a single `pyproject.toml`. The structure looks like this: ```bash @@ -113,8 +218,13 @@ The structure looks like this: The key differences are as follows: 1. `apps/` and `libs/` contain only pure Python code, structured into modules under a common namespace. 2. Tests are colocated with Python code (this will be familiar to those coming from Go or Rust). -3. Because `apps/` is just pure Python code, we need somewhere else to convert this into deployable artifacts (Docker images and the like). So we add `projects/` directory. This contains no code, just a pyproject.toml and whatever else is needed to deploy the built project. The pyproject will specify which internal dependencies are used in the project: exactly one app, and zero or more libs. -4. It must also specify all external dependencies that are used, including the transitive dependencies of internal libs that it uses. But the una CLI will help with this! +3. Because `apps/` is just pure Python code, we need somewhere else to convert this into deployable artifacts (Docker images and the like). +So we add `projects/` directory. This contains no code, just a pyproject.toml and whatever else is needed to deploy the built project. +The pyproject will specify which internal dependencies are used in the project: exactly one app, and zero or more libs. +4. It must also specify all external dependencies that are used, including the transitive dependencies of internal libs that it uses. +But the una CLI will help with this! And there's one more benefit: -5. You can run pyright and pytest from the root directory! This gives you a true monorepo benefit of having a single static analysis of the entire codebase. But don't worry, una will help you to only test the bits that are needed. +5. You can run pyright and pytest from the root directory! +This gives you a true monorepo benefit of having a single static analysis of the entire codebase. +But don't worry, una will help you to only test the bits that are needed. diff --git a/pyproject.toml b/pyproject.toml index 9e84a15..f08f342 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ ] dependencies = [ - "dataclasses-json ~= 0.6.7", + "dataclasses-json ~= 0.6", "rich ~= 13.1", "tomlkit ~= 0.10", "typer ~= 0.8", diff --git a/una/check.py b/una/check.py index ff862d6..8e13eca 100644 --- a/una/check.py +++ b/una/check.py @@ -8,7 +8,7 @@ def check_int_ext_deps(root: Path, ns: str, project: Proj, options: Options) -> bool: - name = project.name + name = config.sanitise_name(project.name) int_dep_imports, ext_dep_imports = collect_all_imports(root, ns, project) collected_libs = collect_known_aliases(project, options) details = create_report( @@ -151,7 +151,8 @@ def create_report( lib_pkgs = {c for c in project.int_deps.libs} int_dep_diff = imports_diff(int_dep_imports, app_pkgs, lib_pkgs) - style = config.get_style() + root = config.get_workspace_root() + style = config.get_style(root) include_libs = style == Style.modules ext_dep_diff = external_deps.calculate_diff(ext_dep_imports, third_party_libs, include_libs) diff --git a/una/cli.py b/una/cli.py index 70e1b38..e50ae90 100644 --- a/una/cli.py +++ b/una/cli.py @@ -44,7 +44,7 @@ def info_command( ): """Info about the Una workspace.""" root = config.get_workspace_root() - ns = config.get_ns() + ns = config.get_ns(root) options = Options(alias=str.split(alias, ",") if alias else []) internal_deps.int_deps_from_projects(root, ns) @@ -79,7 +79,7 @@ def sync_command( ): """Update pyproject.toml with missing int_deps.""" root = config.get_workspace_root() - ns = config.get_ns() + ns = config.get_ns(root) projects = internal_deps.get_projects_data(root, ns) options = Options( quiet=quiet, @@ -104,7 +104,7 @@ def lib_command( ): """Creates an Una lib.""" root = config.get_workspace_root() - style = config.get_style() + style = config.get_style(root) files.create_app_or_lib(root, name, "lib", style) console = Console(theme=defaults.una_theme) console.print("Success!") @@ -117,7 +117,7 @@ def app_command( ): """Creates an Una app.""" root = config.get_workspace_root() - style = config.get_style() + style = config.get_style(root) files.create_app_or_lib(root, name, "app", style) console = Console(theme=defaults.una_theme) console.print("Success!") @@ -130,7 +130,8 @@ def workspace_command( ): """Creates an Una workspace in the current directory.""" path = Path.cwd() - ns = config.get_ns() + root = config.get_workspace_root() + ns = config.get_ns(root) files.create_workspace(path, ns, style) console = Console(theme=defaults.una_theme) console.print("Success!") diff --git a/una/config.py b/una/config.py index 4c56869..5afb10c 100644 --- a/una/config.py +++ b/una/config.py @@ -53,13 +53,15 @@ def get_project_dependencies(data: Conf) -> ExtDeps: return ExtDeps(items=items, source=defaults.pyproj) -def get_ns() -> str: - path = get_workspace_root() - return load_conf(path).project.name +def sanitise_name(name: str) -> str: + return name.replace("-", "_") -def get_style() -> Style: - path = get_workspace_root() +def get_ns(path: Path) -> str: + return sanitise_name(load_conf(path).project.name) + + +def get_style(path: Path) -> Style: return load_conf(path).tool.una.style diff --git a/una/defaults.py b/una/defaults.py index 9bbc050..45775c2 100644 --- a/una/defaults.py +++ b/una/defaults.py @@ -26,6 +26,7 @@ authors = [] dependencies = [] requires-python = "{python_version}" +dynamic = ["una"] # needed for hatch-una metadata hook to work [build-system] requires = ["hatchling", "hatch-una"] diff --git a/una/differ.py b/una/differ.py index bdb720c..631506e 100644 --- a/una/differ.py +++ b/una/differ.py @@ -16,7 +16,7 @@ def calc_diff(tag_name: str | None, only_int_deps: bool): print("No tags found in repository.") return - ns = config.get_ns() + ns = config.get_ns(root) files = get_files(tag) apps_paths = get_changed_apps(root, files, ns) libs_paths = get_changed_libs(root, files, ns) diff --git a/una/external_deps.py b/una/external_deps.py index a6ede83..448c043 100644 --- a/una/external_deps.py +++ b/una/external_deps.py @@ -26,7 +26,7 @@ def external_deps_from_all( def missing_libs(project: Proj, imports: dict[str, OrgImports], options: Options) -> bool: - name = project.name + name = config.sanitise_name(project.name) deps = project.ext_deps int_dep_imports = imports[name] libs = distributions.known_aliases_and_sub_dependencies(deps, options.alias) @@ -59,7 +59,8 @@ def extract_ext_dep_imports(all_imports: Imports, top_ns: str) -> Imports: def _get_third_party_imports(root: Path, paths: set[Path]) -> Imports: - top_ns = config.get_ns() + root = config.get_workspace_root() + top_ns = config.get_ns(root) all_imports = parse.fetch_all_imports(paths) return extract_ext_dep_imports(all_imports, top_ns) @@ -121,7 +122,8 @@ def print_missing_installed_libs( third_party_libs: set[str], project_name: str, ) -> bool: - style = config.get_style() + root = config.get_workspace_root() + style = config.get_style(root) include_libs = style == Style.modules diff = calculate_diff(int_dep_imports, third_party_libs, include_libs) if not diff: diff --git a/una/files.py b/una/files.py index 7a6f2f7..fd33d6f 100644 --- a/una/files.py +++ b/una/files.py @@ -26,7 +26,7 @@ def is_int_dep_dir(p: Path) -> bool: def get_libs_dirs(root: Path, top_dir: str, ns: str) -> list[Path]: - style = config.get_style() + style = config.get_style(root) sub = "" if style == Style.packages else ns lib_dir = root / top_dir / sub if not lib_dir.exists(): @@ -109,7 +109,7 @@ def create_project(path: Path, name: str) -> None: def create_package(path: Path, name: str, kind: Literal["app", "lib"]) -> None: conf = config.load_conf(path) python_version = conf.project.requires_python - ns = conf.project.name + ns = config.get_ns(path) top_dir = defaults.apps_dir if kind == "app" else defaults.libs_dir app_dir = create_dir(path, f"{top_dir}/{name}") @@ -130,9 +130,7 @@ def create_package(path: Path, name: str, kind: Literal["app", "lib"]) -> None: def create_module(path: Path, name: str, kind: Literal["app", "lib"]) -> None: - conf = config.load_conf(path) - ns = conf.project.name - + ns = config.get_ns(path) top_dir = defaults.apps_dir if kind == "app" else defaults.libs_dir code_dir = create_dir(path, f"{top_dir}/{ns}/{name}") create_file(code_dir, "__init__.py") @@ -151,7 +149,8 @@ def create_app_or_lib(path: Path, name: str, kind: DepKind, style: Style) -> Non def get_project_roots(root: Path) -> list[Path]: - style = config.get_style() + ws_root = config.get_workspace_root() + style = config.get_style(ws_root) prefix = "projects" if style == Style.modules else "apps" return sorted(root.glob(f"{prefix}/*/")) diff --git a/una/internal_deps.py b/una/internal_deps.py index 1b093e1..bc158ca 100644 --- a/una/internal_deps.py +++ b/una/internal_deps.py @@ -119,7 +119,8 @@ def get_project_int_deps( def get_int_deps_in_projects(root: Path, libs_paths: list[str], apps_paths: list[str], namespace: str) -> list[Proj]: packages = files.get_projects(root) - style = config.get_style() + ws_root = config.get_workspace_root() + style = config.get_style(ws_root) add_self = style == Style.packages res = [ Proj( diff --git a/una/sync.py b/una/sync.py index 872bd93..a531bce 100644 --- a/una/sync.py +++ b/una/sync.py @@ -80,7 +80,8 @@ def generate_updated_project(conf: Conf, packages: list[Include]) -> str | None: def to_packages(ns: str, diff: Diff) -> list[Include]: - style = config.get_style() + root = config.get_workspace_root() + style = config.get_style(root) apps_path = "../../apps" libs_path = "../../libs" a = [to_package(ns, b, apps_path, style) for b in diff.apps]