From 9d11661299faf4e5af584af6722d967e408fd04a Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 18 Sep 2024 09:16:35 +0000 Subject: [PATCH 01/22] build: Add new build infrastructure This fixes many problems existing in the Nixpkgs Python infrastructure: - No dependency propagation No leaking of dependencies via PYTHONPATH - No runtime deps at build time This causes much less rebuilds - Support for circular dependencies - No wrapping of Python bin's No strange venv breaking shell wrapper - Proper virtualenvs only Meaning that applications that spawns children using sys.executable works fine. - Hermetic bootstrap Nixpkgs breaks if you override any bootstrap dependencies with wheels. - Manual cross splicing Meaning you can override build-time and runtime deps separately, and also IMHO just in general a much better cross interface. --- build/bootstrap.nix | 63 ++++++ build/default.nix | 116 +++++++++++ build/hooks/default.nix | 152 +++++++++++++++ build/hooks/make-venv.py | 181 ++++++++++++++++++ build/hooks/meta-hook.sh | 1 + build/hooks/pyproject-bootstrap-build-hook.sh | 19 ++ build/hooks/pyproject-build-hook.sh | 18 ++ build/hooks/pyproject-bytecode-hook.sh | 19 ++ build/hooks/pyproject-configure-hook.sh | 22 +++ build/hooks/pyproject-install-hook.sh | 26 +++ build/hooks/pyproject-make-venv-hook.sh | 16 ++ build/hooks/pyproject-output-setup-hook.sh | 12 ++ build/hooks/pyproject-pypa-install-hook.sh | 26 +++ build/pkgs/babel/default.nix | 22 +++ build/pkgs/build/default.nix | 34 ++++ build/pkgs/calver/default.nix | 23 +++ build/pkgs/cffi/default.nix | 37 ++++ build/pkgs/cmake/default.nix | 24 +++ build/pkgs/cython/default.nix | 25 +++ build/pkgs/default.nix | 26 +++ build/pkgs/deprecation/default.nix | 26 +++ build/pkgs/distro/default.nix | 22 +++ build/pkgs/dunamai/default.nix | 32 ++++ build/pkgs/flit-core/default.nix | 17 ++ build/pkgs/flit-scm/default.nix | 29 +++ .../pkgs/hatch-fancy-pypi-readme/default.nix | 26 +++ build/pkgs/hatch-jupyter-builder/default.nix | 22 +++ build/pkgs/hatch-nodejs-version/default.nix | 22 +++ build/pkgs/hatch-requirements-txt/default.nix | 27 +++ build/pkgs/hatch-vcs/default.nix | 27 +++ build/pkgs/hatchling/default.nix | 31 +++ build/pkgs/importlib-metadata/default.nix | 34 ++++ build/pkgs/installer/default.nix | 22 +++ build/pkgs/jinja2/default.nix | 34 ++++ build/pkgs/jupyter-packaging/default.nix | 29 +++ build/pkgs/manifestoo-core/default.nix | 33 ++++ build/pkgs/markupsafe/default.nix | 22 +++ build/pkgs/meson-python/default.nix | 33 ++++ build/pkgs/meson/default.nix | 25 +++ build/pkgs/ninja/default.nix | 25 +++ build/pkgs/packaging/default.nix | 22 +++ build/pkgs/pathspec/default.nix | 22 +++ build/pkgs/pbr/default.nix | 22 +++ build/pkgs/pdm-backend/default.nix | 28 +++ build/pkgs/pdm-pep517/default.nix | 16 ++ build/pkgs/pip/default.nix | 24 +++ build/pkgs/pkgconfig/default.nix | 31 +++ build/pkgs/pluggy/default.nix | 23 +++ build/pkgs/poetry-core/default.nix | 17 ++ .../poetry-dynamic-versioning/default.nix | 29 +++ build/pkgs/pybind11/default.nix | 35 ++++ build/pkgs/pyproject-hooks/default.nix | 23 +++ build/pkgs/pyproject-metadata/default.nix | 26 +++ build/pkgs/scikit-build/default.nix | 41 ++++ build/pkgs/setuptools-git/default.nix | 25 +++ build/pkgs/setuptools-scm/default.nix | 53 +++++ build/pkgs/setuptools/default.nix | 24 +++ build/pkgs/toml/default.nix | 22 +++ build/pkgs/tomli/default.nix | 22 +++ build/pkgs/tomlkit/default.nix | 22 +++ build/pkgs/trove-classifiers/default.nix | 23 +++ build/pkgs/typing-extensions/default.nix | 22 +++ build/pkgs/wheel/default.nix | 22 +++ build/pkgs/whool/default.nix | 35 ++++ build/pkgs/zipp/default.nix | 23 +++ build/resolvers.nix | 109 +++++++++++ 66 files changed, 2161 insertions(+) create mode 100644 build/bootstrap.nix create mode 100644 build/default.nix create mode 100644 build/hooks/default.nix create mode 100644 build/hooks/make-venv.py create mode 100644 build/hooks/meta-hook.sh create mode 100644 build/hooks/pyproject-bootstrap-build-hook.sh create mode 100644 build/hooks/pyproject-build-hook.sh create mode 100644 build/hooks/pyproject-bytecode-hook.sh create mode 100644 build/hooks/pyproject-configure-hook.sh create mode 100644 build/hooks/pyproject-install-hook.sh create mode 100644 build/hooks/pyproject-make-venv-hook.sh create mode 100644 build/hooks/pyproject-output-setup-hook.sh create mode 100644 build/hooks/pyproject-pypa-install-hook.sh create mode 100644 build/pkgs/babel/default.nix create mode 100644 build/pkgs/build/default.nix create mode 100644 build/pkgs/calver/default.nix create mode 100644 build/pkgs/cffi/default.nix create mode 100644 build/pkgs/cmake/default.nix create mode 100644 build/pkgs/cython/default.nix create mode 100644 build/pkgs/default.nix create mode 100644 build/pkgs/deprecation/default.nix create mode 100644 build/pkgs/distro/default.nix create mode 100644 build/pkgs/dunamai/default.nix create mode 100644 build/pkgs/flit-core/default.nix create mode 100644 build/pkgs/flit-scm/default.nix create mode 100644 build/pkgs/hatch-fancy-pypi-readme/default.nix create mode 100644 build/pkgs/hatch-jupyter-builder/default.nix create mode 100644 build/pkgs/hatch-nodejs-version/default.nix create mode 100644 build/pkgs/hatch-requirements-txt/default.nix create mode 100644 build/pkgs/hatch-vcs/default.nix create mode 100644 build/pkgs/hatchling/default.nix create mode 100644 build/pkgs/importlib-metadata/default.nix create mode 100644 build/pkgs/installer/default.nix create mode 100644 build/pkgs/jinja2/default.nix create mode 100644 build/pkgs/jupyter-packaging/default.nix create mode 100644 build/pkgs/manifestoo-core/default.nix create mode 100644 build/pkgs/markupsafe/default.nix create mode 100644 build/pkgs/meson-python/default.nix create mode 100644 build/pkgs/meson/default.nix create mode 100644 build/pkgs/ninja/default.nix create mode 100644 build/pkgs/packaging/default.nix create mode 100644 build/pkgs/pathspec/default.nix create mode 100644 build/pkgs/pbr/default.nix create mode 100644 build/pkgs/pdm-backend/default.nix create mode 100644 build/pkgs/pdm-pep517/default.nix create mode 100644 build/pkgs/pip/default.nix create mode 100644 build/pkgs/pkgconfig/default.nix create mode 100644 build/pkgs/pluggy/default.nix create mode 100644 build/pkgs/poetry-core/default.nix create mode 100644 build/pkgs/poetry-dynamic-versioning/default.nix create mode 100644 build/pkgs/pybind11/default.nix create mode 100644 build/pkgs/pyproject-hooks/default.nix create mode 100644 build/pkgs/pyproject-metadata/default.nix create mode 100644 build/pkgs/scikit-build/default.nix create mode 100644 build/pkgs/setuptools-git/default.nix create mode 100644 build/pkgs/setuptools-scm/default.nix create mode 100644 build/pkgs/setuptools/default.nix create mode 100644 build/pkgs/toml/default.nix create mode 100644 build/pkgs/tomli/default.nix create mode 100644 build/pkgs/tomlkit/default.nix create mode 100644 build/pkgs/trove-classifiers/default.nix create mode 100644 build/pkgs/typing-extensions/default.nix create mode 100644 build/pkgs/wheel/default.nix create mode 100644 build/pkgs/whool/default.nix create mode 100644 build/pkgs/zipp/default.nix create mode 100644 build/resolvers.nix diff --git a/build/bootstrap.nix b/build/bootstrap.nix new file mode 100644 index 0000000..f3fef58 --- /dev/null +++ b/build/bootstrap.nix @@ -0,0 +1,63 @@ +{ + stdenv, + lib, + python, + pyprojectInstallHook, + pyprojectBytecodeHook, + pyprojectOutputSetupHook, + python3Packages, +}: + +let + + buildBootstrapPackage = + base: attrs: + stdenv.mkDerivation ( + { + inherit (base) + pname + src + patches + version + meta + ; + dontConfigure = true; + nativeBuildInputs = [ + python + pyprojectInstallHook + pyprojectBytecodeHook + pyprojectOutputSetupHook + ]; + buildPhase = '' + runHook preBuild + + PYTHONPATH="${bootstrap.flit-core}/${python.sitePackages}" \ + ${python.interpreter} -m flit_core.wheel + + runHook postBuild + ''; + } + // attrs + ); + + bootstrap = { + flit-core = buildBootstrapPackage python3Packages.flit-core { + sourceRoot = "${python3Packages.flit-core.src.name}/flit_core"; + buildPhase = '' + runHook preBuild + ${python.interpreter} -m flit_core.wheel + runHook postBuild + ''; + }; + pyproject-hooks = buildBootstrapPackage python3Packages.pyproject-hooks { }; + packaging = buildBootstrapPackage python3Packages.packaging { }; + build = buildBootstrapPackage python3Packages.build { + passthru.dependencies = { + packaging = [ ]; + pyproject-hooks = [ ]; + }; + }; + }; + +in +bootstrap diff --git a/build/default.nix b/build/default.nix new file mode 100644 index 0000000..7894eae --- /dev/null +++ b/build/default.nix @@ -0,0 +1,116 @@ +{ + lib, + python, + newScope, + buildPackages, + stdenv, +}: + +let + inherit (lib) makeScope; + + resolvers = import ./resolvers.nix { inherit lib; }; + inherit (resolvers) resolveCyclic resolveNonCyclic; + + mkResolveBuildSystem = + set: + let + resolveNonCyclic' = resolveNonCyclic set; + in + spec: map (name: set.${name}) (resolveNonCyclic' spec); + + mkResolveVirtualEnv = set: spec: map (name: set.${name}) (resolveCyclic set spec); + + pkgsFun = + let + mkPkgs = import ./pkgs { inherit lib; }; + in + final: mkPkgs { inherit (final) callPackage pyprojectBootstrapHook; }; + +in +makeScope newScope ( + final: + let + mkPythonSet = + { + newScope, + python, + stdenv, + pythonPackagesBuildHost, + bootstrapHooks, + pythonPackagesFun, + }: + makeScope newScope ( + pkgsFinal: + { + inherit python stdenv pythonPackagesBuildHost; + + # Pyproject hook used for bootstrap packages + pyprojectBootstrapHook = pkgsFinal.pyprojectHook.override { + inherit (bootstrapHooks) pyprojectConfigureHook pyprojectBuildHook; + }; + + # Initialize dependency resolvers + resolveBuildSystem = mkResolveBuildSystem pythonPackagesBuildHost; + resolveVirtualEnv = mkResolveVirtualEnv pkgsFinal; + + hooks = pkgsFinal.callPackage ./hooks { }; + inherit (pkgsFinal.hooks) + pyprojectConfigureHook + pyprojectBuildHook + pyprojectInstallHook + pyprojectBytecodeHook + pyprojectOutputSetupHook + pyprojectHook + ; + } + // pythonPackagesFun pkgsFinal + ); + + bootstrapHooks = final.callPackage ./hooks { + python = final.python.pythonOnBuildForHost; + resolveBuildSystem = mkResolveBuildSystem final.pythonPackagesBootstrap; + hooks = bootstrapHooks; + }; + + in + { + # Allows overriding Python by calling overrideScope on the outer scope + inherit python; + + pythonPackagesBootstrap = mkPythonSet { + inherit (buildPackages) stdenv newScope; + inherit bootstrapHooks; + python = python.pythonOnBuildForHost; + pythonPackagesBuildHost = final.pythonPackagesBootstrap; + pythonPackagesFun = + _: + final.callPackage ./bootstrap.nix { + inherit (bootstrapHooks) pyprojectInstallHook pyprojectBytecodeHook pyprojectOutputSetupHook; + python = final.python.pythonOnBuildForHost; + }; + }; + + # Python packages for the build host + pythonPackagesBuildHost = mkPythonSet { + inherit (buildPackages) stdenv newScope; + python = python.pythonOnBuildForHost; + inherit (final) pythonPackagesBuildHost; + bootstrapHooks = final.pythonPackagesBootstrap.hooks; + pythonPackagesFun = pkgsFun; + }; + + # Python packages for the target host + pythonPackagesHostHost = + # If we're not doing cross reference build host packages + if stdenv.buildPlatform != stdenv.hostPlatform then + mkPythonSet { + inherit (final) newScope pythonPackagesBuildHost; + inherit python stdenv; + bootstrapHooks = final.pythonPackagesBuildHost.hooks; + pythonPackagesFun = pkgsFun; + } + else + final.pythonPackagesBuildHost; + } +) diff --git a/build/hooks/default.nix b/build/hooks/default.nix new file mode 100644 index 0000000..b93e0b2 --- /dev/null +++ b/build/hooks/default.nix @@ -0,0 +1,152 @@ +{ + callPackage, + makeSetupHook, + python, + pkgs, + lib, + resolveBuildSystem, + stdenv, + hooks, + pythonPackagesBuildHost, +}: +let + inherit (python) pythonOnBuildForHost isPy3k; + inherit (pkgs) buildPackages; + pythonInterpreter = pythonOnBuildForHost.interpreter; + pythonSitePackages = python.sitePackages; + +in +{ + # Build hook used to build PEP-621/setuptools projects + pyprojectConfigureHook = callPackage ( + { python }: + makeSetupHook { + name = "pyproject-configure-hook"; + substitutions = { + inherit pythonInterpreter; + pythonPath = lib.concatStringsSep ":" ( + lib.optional ( + stdenv.buildPlatform != stdenv.hostPlatform + ) "${python.pythonOnBuildForHost}/${python.sitePackages}" + ++ [ + "${python}/${python.sitePackages}" + ] + ); + }; + } ./pyproject-configure-hook.sh + ) { }; + + # Build hook used to build PEP-621/setuptools projects + pyprojectBuildHook = callPackage ( + _: + makeSetupHook { + name = "pyproject-build-hook"; + substitutions = { + inherit (pythonPackagesBuildHost) build; + inherit pythonInterpreter; + }; + propagatedBuildInputs = resolveBuildSystem { + build = [ ]; + }; + } ./pyproject-build-hook.sh + ) { }; + + pyprojectInstallHook = + callPackage + ( + { uv }: + makeSetupHook { + name = "pyproject-install-hook"; + substitutions = { + inherit pythonInterpreter uv; + }; + } ./pyproject-install-hook.sh + ) + { + inherit (buildPackages) uv; + }; + + pyprojectPypaInstallHook = callPackage ( + { pythonPackagesBuildHost }: + makeSetupHook { + name = "pyproject-pypa-install-hook"; + substitutions = { + inherit (pythonPackagesBuildHost) installer; + inherit pythonInterpreter pythonSitePackages; + }; + } ./pyproject-pypa-install-hook.sh + ) { }; + + pyprojectBytecodeHook = callPackage ( + _: + makeSetupHook { + name = "pyproject-bytecode-hook"; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + compileArgs = lib.concatStringsSep " " ( + [ + "-q" + "-f" + "-i -" + ] + ++ lib.optionals isPy3k [ "-j $NIX_BUILD_CORES" ] + ); + bytecodeName = if isPy3k then "__pycache__" else "*.pyc"; + }; + } ./pyproject-bytecode-hook.sh + ) { }; + + pyprojectOutputSetupHook = callPackage ( + _: + makeSetupHook { + name = "pyproject-output-setup-hook"; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pyproject-output-setup-hook.sh + ) { }; + + pyprojectMakeVenvHook = callPackage ( + { python }: + makeSetupHook { + name = "pyproject-make-venv-hook"; + substitutions = { + inherit pythonInterpreter python; + makeVenvScript = ./make-venv.py; + }; + } ./pyproject-make-venv-hook.sh + ) { }; + + # Meta hook aggregating the default pyproject.toml/setup.py install behaviour and adds Python + pyprojectHook = + callPackage + ( + { + pyprojectConfigureHook, + pyprojectBuildHook, + pyprojectInstallHook, + pyprojectOutputSetupHook, + python, + }: + makeSetupHook { + name = "pyproject-hook"; + propagatedBuildInputs = [ + python + pyprojectConfigureHook + pyprojectBuildHook + pyprojectInstallHook + pyprojectOutputSetupHook + ]; + } ./meta-hook.sh + ) + ( + { + python = pythonOnBuildForHost; + } + // (lib.optionalAttrs (stdenv.buildPlatform != stdenv.hostPlatform) { + # Uv is not yet compatible with cross installs, or at least I can't figure out the magic incantation. + # We can use installer for cross, and still use uv for native. + pyprojectInstallHook = hooks.pyprojectPypaInstallHook; + }) + ); +} diff --git a/build/hooks/make-venv.py b/build/hooks/make-venv.py new file mode 100644 index 0000000..37141ce --- /dev/null +++ b/build/hooks/make-venv.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +from venv import EnvBuilder +from pathlib import Path +import os.path +import shutil +import stat +import sys +import os + + +EXECUTABLE = os.path.basename(sys.executable) +PYTHON_VERSION = ".".join((str(sys.version_info.major), str(sys.version_info.minor))) +SITE_PACKAGES = os.path.join("lib", f"python{PYTHON_VERSION}", "site-packages") + + +# Look for shebangs pointing to Python bin directory, rewrite them to venv directory +dep_shebang = ("#!" + os.path.dirname(sys.executable)).encode() + + +# Special rules for bin directory: +# - Copy symlinks +# - Rewrite scripts for shebangs +# - Symlink anything else +def write_bin_dir(bin_dir: Path, bin_out: Path) -> None: + out_shebang: bytes = f"#!{bin_out}".encode() + + for bin in os.listdir(bin_dir): + bin_file = bin_dir.joinpath(bin) + bin_file_out = bin_out.joinpath(bin_file.name) + st_mode = bin_file.lstat().st_mode + + # Copy symlinks + if stat.S_ISLNK(st_mode): + shutil.copy(bin_file, bin_out.joinpath(bin_file.name), follow_symlinks=False) + + # Rewrite script shebangs + elif stat.S_ISREG(st_mode): + # Check if file starts with Python shebang + with open(bin_file, "rb") as f_in: + shebang = f_in.read(len(dep_shebang)) + # If it does, rewrite it to venv interpreter + if shebang == dep_shebang: + with open(bin_file_out, "wb") as f_out: + f_out.write(out_shebang) + shutil.copyfileobj(f_in, f_out) + os.chmod(bin_file_out, st_mode) # Copy mode + continue + + # Symlink anything else + os.symlink(bin_file, bin_file_out) + + +# Special rules for site-packages: +# - Rewrite package-local symlinks to venv +# - Directly symlink regardless of type +def write_site_packages(site_packages: Path, out_site_packages: Path) -> None: + for site_pkg_file in os.listdir(site_packages): + src_file = site_packages.joinpath(site_pkg_file) + dst_file = out_site_packages.joinpath(site_pkg_file) + if stat.S_ISLNK(src_file.lstat().st_mode): + shutil.copy(src_file, dst_file, follow_symlinks=False) + else: + os.symlink(src_file, dst_file) + + +def link_dependency(dep_root: Path, out_root: Path) -> None: + site_packages = dep_root.joinpath(SITE_PACKAGES) + bin_dir = dep_root.joinpath("bin") + nix_support_dir = dep_root.joinpath("nix-support") + + def _link(root: Path, out: Path) -> None: + # Let other hooks manage the nix-support + if root == nix_support_dir: + return + + try: + os.mkdir(out) + except FileExistsError: + pass + + for filename in os.listdir(root): + path = root.joinpath(filename) + + # Special case handle bin/site-packages + if path == bin_dir: + write_bin_dir(path, out.joinpath("bin")) + continue + elif path == site_packages: + write_site_packages(path, out_root.joinpath(SITE_PACKAGES)) + continue + + st_mode = path.lstat().st_mode + + if stat.S_ISLNK(st_mode): + shutil.copy(path, out.joinpath(path.name)) + elif stat.S_ISREG(st_mode): + os.symlink(path, out.joinpath(path.name)) + elif stat.S_ISDIR(st_mode): + _link(path, out.joinpath(path.name)) + else: + raise ValueError(f"Unhandled st_mode: {st_mode}") + + _link(dep_root, out_root) + + +def unique_strings(input: list[str]) -> list[str]: + ret: list[str] = [] + for i in input: + if i not in ret: + ret.append(i) + return ret + + +def fixup_pyvenv(python_root: Path, out_root: Path) -> None: + # The venv module writes a command line to pyvenv.cfg using sys.executable + # This means that the output would contain a reference to build Python + with open(out_root.joinpath("pyvenv.cfg"), "r") as pyvenv_f: + pyvenv = pyvenv_f.read() + + pyvenv = pyvenv.replace(os.path.dirname(os.path.dirname(sys.executable)), str(python_root)) + + with open(out_root.joinpath("pyvenv.cfg"), "w") as pyvenv_f: + pyvenv_f.write(pyvenv) + + +def main(): + import argparse + + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("out", help="Virtualenv output directory") + arg_parser.add_argument("--python", help="Python to link virtualenv to", default=os.path.dirname(os.path.dirname(sys.executable))) + arg_parser.add_argument("--env", action="append", help="Source dependencies from environment variable") + arg_parser.add_argument("--deps", action="append", help="Source dependencies from colon separated list") + + args = arg_parser.parse_args() + + out_root = Path(args.out) + python_root = Path(args.python) + python_executable = python_root.joinpath("bin", EXECUTABLE) + + dependencies: list[Path] = [] # List of dependency roots + seen_roots: set[str] = set() # Keep track of unique dependency roots + + # Populate dependencies from precisely passed options + for dep_roots in (args.deps or []): + for dep_root in dep_roots.split(":"): + if dep_root in seen_roots: + continue + seen_roots.add(dep_root) + dependencies.append(Path(dep_root)) + + # Populate dependencies from env + for env_var in (args.env or []): + try: + env_value = os.environ[env_var] + except KeyError: + continue + + for dep_root in env_value.split(":"): + if dep_root in seen_roots: + continue + seen_roots.add(dep_root) + dependencies.append(Path(dep_root)) + + # Write virtualenv + builder = EnvBuilder(symlinks=True) + context = builder.ensure_directories(str(out_root)) + context.executable = str(python_executable) + context.python_dir = str(python_root.joinpath("bin")) + builder.setup_scripts(context) + builder.setup_python(context) + builder.create_configuration(context) + fixup_pyvenv(python_root, out_root) + + # Link packages into env + for dep_root in dependencies: + link_dependency(dep_root, out_root) + + +if __name__ == "__main__": + main() diff --git a/build/hooks/meta-hook.sh b/build/hooks/meta-hook.sh new file mode 100644 index 0000000..02725e6 --- /dev/null +++ b/build/hooks/meta-hook.sh @@ -0,0 +1 @@ +# Dummy hook to create a meta hook only consisting of other hooks diff --git a/build/hooks/pyproject-bootstrap-build-hook.sh b/build/hooks/pyproject-bootstrap-build-hook.sh new file mode 100644 index 0000000..d3ac37c --- /dev/null +++ b/build/hooks/pyproject-bootstrap-build-hook.sh @@ -0,0 +1,19 @@ +# Setup hook to use for PEP-621/setuptools builds +echo "Sourcing pyproject-build-hook" + +pyprojectBuildPhase() { + echo "Executing pyprojectBuildPhase" + runHook preBuild + + echo "Creating a wheel..." + @build@/bin/pyproject-build --no-isolation --outdir dist/ --wheel $pyprojectBuildFlags + echo "Finished creating a wheel..." + + runHook postBuild + echo "Finished executing pyprojectBuildPhase" +} + +if [ -z "${dontUsePyprojectBuild-}" ] && [ -z "${buildPhase-}" ]; then + echo "Using pyprojectBuildPhase" + buildPhase=pyprojectBuildPhase +fi diff --git a/build/hooks/pyproject-build-hook.sh b/build/hooks/pyproject-build-hook.sh new file mode 100644 index 0000000..f8f4ad3 --- /dev/null +++ b/build/hooks/pyproject-build-hook.sh @@ -0,0 +1,18 @@ +# Setup hook to use for PEP-621/setuptools builds +echo "Sourcing pyproject-build-hook" + +pyprojectBuildPhase() { + echo "Executing pyprojectBuildPhase" + runHook preBuild + + echo "Creating a wheel..." + env PYTHONPATH="${NIX_PYPROJECT_PYTHONPATH}:${PYTHONPATH}" @build@/bin/pyproject-build --no-isolation --outdir dist/ --wheel $pypaBuildFlags + + runHook postBuild + echo "Finished executing pyprojectBuildPhase" +} + +if [ -z "${dontUsePyprojectBuild-}" ] && [ -z "${buildPhase-}" ]; then + echo "Using pyprojectBuildPhase" + buildPhase=pyprojectBuildPhase +fi diff --git a/build/hooks/pyproject-bytecode-hook.sh b/build/hooks/pyproject-bytecode-hook.sh new file mode 100644 index 0000000..48459e8 --- /dev/null +++ b/build/hooks/pyproject-bytecode-hook.sh @@ -0,0 +1,19 @@ +pyprojectBytecodePhase () { + if [ -d "$out/bin" ]; then + rm -rf "$out/bin/__pycache__" # Python 3 + find "$out/bin" -type f -name "*.pyc" -delete # Python 2 + fi + + items="$(find "$out" -name "@bytecodeName@")" + if [[ -n $items ]]; then + for pycache in $items; do + rm -rf "$pycache" + done + fi + + @pythonInterpreter@ -OO -m compileall @compileArgs@ "$out"/@pythonSitePackages@ +} + +if [ -z "${dontUsePyprojectBytecode-}" ]; then + postPhases+=" pyprojectBytecodePhase" +fi diff --git a/build/hooks/pyproject-configure-hook.sh b/build/hooks/pyproject-configure-hook.sh new file mode 100644 index 0000000..86f2775 --- /dev/null +++ b/build/hooks/pyproject-configure-hook.sh @@ -0,0 +1,22 @@ +# Setup hook to use for PEP-621/setuptools builds +echo "Sourcing pyproject-configure-hook" + +pyprojectConfigurePhase() { + echo "Executing pyprojectConfigurePhase" + runHook preConfigure + + # Undo any Python dependency propagation leaking into build, and set it to our interpreters PYTHONPATH + # + # In case of cross compilation this variable will contain two entries: + # One for the native Python and one for the cross built, so the native can load sysconfig + # information from the cross compiled Python. + export PYTHONPATH=@pythonPath@ + + runHook postConfigure + echo "Finished executing pyprojectConfigurePhase" +} + +if [ -z "${dontUsePyprojectConfigure-}" ] && [ -z "${configurePhase-}" ]; then + echo "Using pyprojectConfiguredPhase" + configurePhase=pyprojectConfigurePhase +fi diff --git a/build/hooks/pyproject-install-hook.sh b/build/hooks/pyproject-install-hook.sh new file mode 100644 index 0000000..599d063 --- /dev/null +++ b/build/hooks/pyproject-install-hook.sh @@ -0,0 +1,26 @@ +# Setup hook for Pyproject installer. +echo "Sourcing pyproject-install-hook" + +pyprojectInstallPhase() { + echo "Executing pyprojectInstallPhase" + runHook preInstall + + pushd dist > /dev/null + + for wheel in ./*.whl; do + @uv@/bin/uv pip --offline --no-cache install --no-deps --link-mode=copy --system --prefix "$out" $uvPipInstallFlags "$wheel" + echo "Successfully installed $wheel" + done + + rm -f "$out/.lock" + + popd > /dev/null + + runHook postInstall + echo "Finished executing pyprojectInstallPhase" +} + +if [ -z "${dontUsePyprojectInstall-}" ] && [ -z "${installPhase-}" ]; then + echo "Using pyprojectInstallPhase" + installPhase=pyprojectInstallPhase +fi diff --git a/build/hooks/pyproject-make-venv-hook.sh b/build/hooks/pyproject-make-venv-hook.sh new file mode 100644 index 0000000..1b3952b --- /dev/null +++ b/build/hooks/pyproject-make-venv-hook.sh @@ -0,0 +1,16 @@ +echo "Sourcing pyproject-make-venv-hook" + +pyprojectMakeVenv() { + echo "Executing pyprojectMakeVenv" + runHook preInstall + + @pythonInterpreter@ @makeVenvScript@ --python @python@ "$out" --env "NIX_PYPROJECT_DEPS" + + runHook postInstall + echo "Finished executing pyprojectMakeVenv" +} + +if [ -z "${dontUsePyprojectMakeVenv-}" ] && [ -z "${installPhase-}" ]; then + echo "Using pyprojectMakeVenv" + installPhase=pyprojectMakeVenv +fi diff --git a/build/hooks/pyproject-output-setup-hook.sh b/build/hooks/pyproject-output-setup-hook.sh new file mode 100644 index 0000000..28a324c --- /dev/null +++ b/build/hooks/pyproject-output-setup-hook.sh @@ -0,0 +1,12 @@ +pyprojectOutputSetupHook () { +mkdir -p $out/nix-support +cat >> $out/nix-support/setup-hook << EOF +# Add Python dependency to search path for discovery by build +addToSearchPath NIX_PYPROJECT_DEPS "$out" +addToSearchPath NIX_PYPROJECT_PYTHONPATH "$out/@pythonSitePackages@" +EOF +} + +if [ -z "${dontUsePyprojectOutputSetupHook-}" ]; then + postPhases+=" pyprojectOutputSetupHook" +fi diff --git a/build/hooks/pyproject-pypa-install-hook.sh b/build/hooks/pyproject-pypa-install-hook.sh new file mode 100644 index 0000000..d523919 --- /dev/null +++ b/build/hooks/pyproject-pypa-install-hook.sh @@ -0,0 +1,26 @@ +# Setup hook for Pyproject installer. +echo "Sourcing pyproject-pypa-install-hook" + +pyprojectPypaInstallPhase() { + echo "Executing pyprojectPypaInstallPhase" + runHook preInstall + + pushd dist > /dev/null + + for wheel in *.whl; do + env PYTHONPATH=$PYTHONPATH:@installer@/@pythonSitePackages@ @pythonInterpreter@ -m installer --prefix "$out" "$wheel" + echo "Successfully installed $wheel" + done + + rm -f "$out/.lock" + + popd > /dev/null + + runHook postInstall + echo "Finished executing pyprojectPypaInstallPhase" +} + +if [ -z "${dontUsePyprojectInstall-}" ] && [ -z "${installPhase-}" ]; then + echo "Using pyprojectPypaInstallPhase" + installPhase=pyprojectPypaInstallPhase +fi diff --git a/build/pkgs/babel/default.nix b/build/pkgs/babel/default.nix new file mode 100644 index 0000000..18e3b7c --- /dev/null +++ b/build/pkgs/babel/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.babel) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/build/default.nix b/build/pkgs/build/default.nix new file mode 100644 index 0000000..783d1a3 --- /dev/null +++ b/build/pkgs/build/default.nix @@ -0,0 +1,34 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.build) + pname + version + src + meta + ; + + passthru.dependencies = + { + packaging = [ ]; + pyproject-hooks = [ ]; + + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/calver/default.nix b/build/pkgs/calver/default.nix new file mode 100644 index 0000000..adcae3f --- /dev/null +++ b/build/pkgs/calver/default.nix @@ -0,0 +1,23 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.calver) + pname + version + src + meta + postPatch + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/cffi/default.nix b/build/pkgs/cffi/default.nix new file mode 100644 index 0000000..e8c2e8f --- /dev/null +++ b/build/pkgs/cffi/default.nix @@ -0,0 +1,37 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, + pkg-config, + libffi, +}: +stdenv.mkDerivation { + inherit (python3Packages.cffi) + pname + version + src + meta + patches + postPatch + ; + + env = + { + inherit (python3Packages.cffi) NIX_CFLAGS_COMPILE; + }; + + buildInputs = [ libffi ]; + + nativeBuildInputs = + [ + pyprojectHook + pkg-config + python + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/cmake/default.nix b/build/pkgs/cmake/default.nix new file mode 100644 index 0000000..84a2e6e --- /dev/null +++ b/build/pkgs/cmake/default.nix @@ -0,0 +1,24 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.cmake) + pname + version + src + meta + postUnpack + setupHooks + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/cython/default.nix b/build/pkgs/cython/default.nix new file mode 100644 index 0000000..1205cf0 --- /dev/null +++ b/build/pkgs/cython/default.nix @@ -0,0 +1,25 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, + pkg-config, +}: +stdenv.mkDerivation { + inherit (python3Packages.cython) + pname + version + src + meta + setupHook + ; + + nativeBuildInputs = + [ + pyprojectHook + pkg-config + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/default.nix b/build/pkgs/default.nix new file mode 100644 index 0000000..78e975c --- /dev/null +++ b/build/pkgs/default.nix @@ -0,0 +1,26 @@ +{ lib }: + +let + # List all packages in directory + paths = lib.filterAttrs (_name: type: type == "directory") (builtins.readDir ./.); + +in +{ callPackage, pyprojectBootstrapHook }: +# Automatically call all packages +(lib.mapAttrs (name: _: callPackage (./. + "/${name}") { }) paths) +// + # Override bootstrap packages with bootstrap hook + { + build = callPackage ./build { + pyprojectHook = pyprojectBootstrapHook; + }; + flit-core = callPackage ./flit-core { + pyprojectHook = pyprojectBootstrapHook; + }; + packaging = callPackage ./packaging { + pyprojectHook = pyprojectBootstrapHook; + }; + pyproject-hooks = callPackage ./pyproject-hooks { + pyprojectHook = pyprojectBootstrapHook; + }; + } diff --git a/build/pkgs/deprecation/default.nix b/build/pkgs/deprecation/default.nix new file mode 100644 index 0000000..00c225e --- /dev/null +++ b/build/pkgs/deprecation/default.nix @@ -0,0 +1,26 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.deprecation) + pname + version + src + meta + ; + + passthru.dependencies = { + packaging = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/distro/default.nix b/build/pkgs/distro/default.nix new file mode 100644 index 0000000..8fb83a0 --- /dev/null +++ b/build/pkgs/distro/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.distro) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/dunamai/default.nix b/build/pkgs/dunamai/default.nix new file mode 100644 index 0000000..bac50cd --- /dev/null +++ b/build/pkgs/dunamai/default.nix @@ -0,0 +1,32 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.dunamai) + pname + version + src + meta + ; + + passthru.dependencies = + { + packaging = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.8") { + importlib-metadata = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + poetry-core = [ ]; + }; +} diff --git a/build/pkgs/flit-core/default.nix b/build/pkgs/flit-core/default.nix new file mode 100644 index 0000000..60ef771 --- /dev/null +++ b/build/pkgs/flit-core/default.nix @@ -0,0 +1,17 @@ +{ + stdenv, + python3Packages, + pyprojectHook, +}: +stdenv.mkDerivation { + inherit (python3Packages.flit-core) + pname + version + src + sourceRoot + meta + ; + nativeBuildInputs = [ + pyprojectHook + ]; +} diff --git a/build/pkgs/flit-scm/default.nix b/build/pkgs/flit-scm/default.nix new file mode 100644 index 0000000..af75ed3 --- /dev/null +++ b/build/pkgs/flit-scm/default.nix @@ -0,0 +1,29 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation (finalAttrs: { + inherit (python3Packages.flit-scm) + pname + version + src + meta + ; + + passthru.dependencies = + { + flit-core = [ ]; + setuptools-scm = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem finalAttrs.passthru.dependencies; +}) diff --git a/build/pkgs/hatch-fancy-pypi-readme/default.nix b/build/pkgs/hatch-fancy-pypi-readme/default.nix new file mode 100644 index 0000000..782a637 --- /dev/null +++ b/build/pkgs/hatch-fancy-pypi-readme/default.nix @@ -0,0 +1,26 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.hatch-fancy-pypi-readme) + pname + version + src + meta + ; + + passthru.dependencies = { + hatchling = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + }; +} diff --git a/build/pkgs/hatch-jupyter-builder/default.nix b/build/pkgs/hatch-jupyter-builder/default.nix new file mode 100644 index 0000000..825b892 --- /dev/null +++ b/build/pkgs/hatch-jupyter-builder/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation (finalAttrs: { + inherit (python3Packages.hatch-jupyter-builder) + pname + version + src + meta + ; + + passthru.dependencies = { + hatchling = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem finalAttrs.passthru.dependencies; +}) diff --git a/build/pkgs/hatch-nodejs-version/default.nix b/build/pkgs/hatch-nodejs-version/default.nix new file mode 100644 index 0000000..c8adccd --- /dev/null +++ b/build/pkgs/hatch-nodejs-version/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation (finalAttrs: { + inherit (python3Packages.hatch-nodejs-version) + pname + version + src + meta + ; + + passthru.dependencies = { + hatchling = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem finalAttrs.passthru.dependencies; +}) diff --git a/build/pkgs/hatch-requirements-txt/default.nix b/build/pkgs/hatch-requirements-txt/default.nix new file mode 100644 index 0000000..c836b9a --- /dev/null +++ b/build/pkgs/hatch-requirements-txt/default.nix @@ -0,0 +1,27 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation (_finalAttrs: { + inherit (python3Packages.hatch-requirements-txt) + pname + version + src + meta + ; + + passthru.dependencies = { + hatchling = [ ]; + packaging = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + }; +}) diff --git a/build/pkgs/hatch-vcs/default.nix b/build/pkgs/hatch-vcs/default.nix new file mode 100644 index 0000000..132c12f --- /dev/null +++ b/build/pkgs/hatch-vcs/default.nix @@ -0,0 +1,27 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation (_finalAttrs: { + inherit (python3Packages.hatch-vcs) + pname + version + src + meta + ; + + passthru.dependencies = { + hatchling = [ ]; + setuptools-scm = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ]; + + build_system = resolveBuildSystem { + hatchling = [ ]; + }; +}) diff --git a/build/pkgs/hatchling/default.nix b/build/pkgs/hatchling/default.nix new file mode 100644 index 0000000..dae4174 --- /dev/null +++ b/build/pkgs/hatchling/default.nix @@ -0,0 +1,31 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation (finalAttrs: { + inherit (python3Packages.hatchling) + pname + version + src + meta + ; + + passthru.dependencies = + { + packaging = [ ]; + pathspec = [ ]; + pluggy = [ ]; + trove-classifiers = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem finalAttrs.passthru.dependencies; +}) diff --git a/build/pkgs/importlib-metadata/default.nix b/build/pkgs/importlib-metadata/default.nix new file mode 100644 index 0000000..f3d9f70 --- /dev/null +++ b/build/pkgs/importlib-metadata/default.nix @@ -0,0 +1,34 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.importlib-metadata) + pname + version + src + meta + ; + + passthru.dependencies = + { + toml = [ ]; + zipp = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.8") { + typing-extensions = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + setuptools-scm = [ ]; + }; +} diff --git a/build/pkgs/installer/default.nix b/build/pkgs/installer/default.nix new file mode 100644 index 0000000..76b2ff4 --- /dev/null +++ b/build/pkgs/installer/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + flit-core, + pythonPackages, + pyprojectHook, + resolveBuildSystem, +}: + +stdenv.mkDerivation { + inherit (pythonPackages.installer) + pname + version + src + meta + ; + + nativeBuildInputs = + [ pyprojectHook ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/jinja2/default.nix b/build/pkgs/jinja2/default.nix new file mode 100644 index 0000000..1bfb7be --- /dev/null +++ b/build/pkgs/jinja2/default.nix @@ -0,0 +1,34 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.jinja2) + pname + version + src + meta + ; + + passthru = { + dependencies = { + markupsafe = [ ]; + }; + + optional-dependencies = { + i18n = { + babel = [ ]; + }; + }; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/jupyter-packaging/default.nix b/build/pkgs/jupyter-packaging/default.nix new file mode 100644 index 0000000..f48db04 --- /dev/null +++ b/build/pkgs/jupyter-packaging/default.nix @@ -0,0 +1,29 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.jupyter-packaging) + pname + version + src + meta + ; + + passthru.dependencies = { + deprecation = [ ]; + packaging = [ ]; + setuptools = [ ]; + tomlkit = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + }; +} diff --git a/build/pkgs/manifestoo-core/default.nix b/build/pkgs/manifestoo-core/default.nix new file mode 100644 index 0000000..34dc93c --- /dev/null +++ b/build/pkgs/manifestoo-core/default.nix @@ -0,0 +1,33 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.manifestoo-core) + pname + version + src + meta + ; + + passthru.dependencies = + lib.optionalAttrs (python.pythonOlder "3.7") { + importlib-resources = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.8") { + typing-extensions = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + hatch-vcs = [ ]; + }; +} diff --git a/build/pkgs/markupsafe/default.nix b/build/pkgs/markupsafe/default.nix new file mode 100644 index 0000000..3b87de9 --- /dev/null +++ b/build/pkgs/markupsafe/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.markupsafe) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/meson-python/default.nix b/build/pkgs/meson-python/default.nix new file mode 100644 index 0000000..46a78c0 --- /dev/null +++ b/build/pkgs/meson-python/default.nix @@ -0,0 +1,33 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: + +stdenv.mkDerivation (finalAttrs: { + inherit (python3Packages.meson-python) + pname + version + src + meta + setupHooks + ; + + passthru.dependencies = + { + meson = [ ]; + ninja = [ ]; + pyproject-metadata = [ ]; + tomli = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.10") { + typing-extensions = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem finalAttrs.passthru.dependencies; +}) diff --git a/build/pkgs/meson/default.nix b/build/pkgs/meson/default.nix new file mode 100644 index 0000000..a7e0018 --- /dev/null +++ b/build/pkgs/meson/default.nix @@ -0,0 +1,25 @@ +{ + stdenv, + pyprojectHook, + resolveBuildSystem, + pkgs, +}: +stdenv.mkDerivation { + inherit (pkgs.meson) + pname + version + src + meta + patches + setupHook + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + wheel = [ ]; + }; +} diff --git a/build/pkgs/ninja/default.nix b/build/pkgs/ninja/default.nix new file mode 100644 index 0000000..02b7784 --- /dev/null +++ b/build/pkgs/ninja/default.nix @@ -0,0 +1,25 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.ninja) + pname + version + src + meta + postUnpack + setupHook + preBuild + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/packaging/default.nix b/build/pkgs/packaging/default.nix new file mode 100644 index 0000000..dfdc261 --- /dev/null +++ b/build/pkgs/packaging/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.packaging) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/pathspec/default.nix b/build/pkgs/pathspec/default.nix new file mode 100644 index 0000000..0b9e272 --- /dev/null +++ b/build/pkgs/pathspec/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pathspec) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/pbr/default.nix b/build/pkgs/pbr/default.nix new file mode 100644 index 0000000..e2028ef --- /dev/null +++ b/build/pkgs/pbr/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pbr) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/pdm-backend/default.nix b/build/pkgs/pdm-backend/default.nix new file mode 100644 index 0000000..144a437 --- /dev/null +++ b/build/pkgs/pdm-backend/default.nix @@ -0,0 +1,28 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, +}: +stdenv.mkDerivation (finalAttrs: { + inherit (python3Packages.pdm-backend) + pname + version + src + meta + setupHook + ; + + env = { + PDM_BUILD_SCM_VERSION = finalAttrs.version; + }; + + passthru.dependencies = lib.optionalAttrs (python.pythonOlder "3.10") { + importlib-metadata = [ ]; + }; + + nativeBuildInputs = [ + pyprojectHook + ]; +}) diff --git a/build/pkgs/pdm-pep517/default.nix b/build/pkgs/pdm-pep517/default.nix new file mode 100644 index 0000000..b3ca4af --- /dev/null +++ b/build/pkgs/pdm-pep517/default.nix @@ -0,0 +1,16 @@ +{ + stdenv, + python3Packages, + pyprojectHook, +}: +stdenv.mkDerivation { + inherit (python3Packages.pdm-pep517) + pname + version + src + meta + ; + nativeBuildInputs = [ + pyprojectHook + ]; +} diff --git a/build/pkgs/pip/default.nix b/build/pkgs/pip/default.nix new file mode 100644 index 0000000..155f2c3 --- /dev/null +++ b/build/pkgs/pip/default.nix @@ -0,0 +1,24 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pip) + pname + version + src + meta + postPatch + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + wheel = [ ]; + }; +} diff --git a/build/pkgs/pkgconfig/default.nix b/build/pkgs/pkgconfig/default.nix new file mode 100644 index 0000000..6a834c5 --- /dev/null +++ b/build/pkgs/pkgconfig/default.nix @@ -0,0 +1,31 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pkgconfig) + pname + version + src + meta + postPatch + ; + + inherit (python3Packages.pkgconfig) + setupHooks + wrapperName + suffixSalt + targetPrefix + baseBinName + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + poetry-core = [ ]; + }; +} diff --git a/build/pkgs/pluggy/default.nix b/build/pkgs/pluggy/default.nix new file mode 100644 index 0000000..8ee9c22 --- /dev/null +++ b/build/pkgs/pluggy/default.nix @@ -0,0 +1,23 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, + setuptools-scm, +}: +stdenv.mkDerivation { + inherit (python3Packages.pluggy) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools-scm = [ ]; + }; +} diff --git a/build/pkgs/poetry-core/default.nix b/build/pkgs/poetry-core/default.nix new file mode 100644 index 0000000..ac595f3 --- /dev/null +++ b/build/pkgs/poetry-core/default.nix @@ -0,0 +1,17 @@ +{ + stdenv, + python3Packages, + pyprojectHook, +}: +stdenv.mkDerivation { + inherit (python3Packages.poetry-core) + pname + version + src + meta + ; + + nativeBuildInputs = [ + pyprojectHook + ]; +} diff --git a/build/pkgs/poetry-dynamic-versioning/default.nix b/build/pkgs/poetry-dynamic-versioning/default.nix new file mode 100644 index 0000000..6d55e80 --- /dev/null +++ b/build/pkgs/poetry-dynamic-versioning/default.nix @@ -0,0 +1,29 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.poetry-dynamic-versioning) + pname + version + src + meta + setupHook + ; + + passthru.dependencies = { + dunamai = [ ]; + jinja2 = [ ]; + tomlkit = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + poetry-core = [ ]; + }; +} diff --git a/build/pkgs/pybind11/default.nix b/build/pkgs/pybind11/default.nix new file mode 100644 index 0000000..6cc6269 --- /dev/null +++ b/build/pkgs/pybind11/default.nix @@ -0,0 +1,35 @@ +{ + stdenv, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pybind11) + pname + version + src + meta + propagatedNativeBuildInputs # setupHook + dontUseCmakeBuildDir + cmakeFlags + hardeningDisable + ; + + postInstall = '' + # Symlink the CMake-installed headers to the location expected by setuptools + mkdir -p $out/include/${python.libPrefix} + ln -sf $out/include/pybind11 $out/include/${python.libPrefix}/pybind11 + ''; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + cmake = [ ]; + ninja = [ ]; + setuptools = [ ]; + }; +} diff --git a/build/pkgs/pyproject-hooks/default.nix b/build/pkgs/pyproject-hooks/default.nix new file mode 100644 index 0000000..b78286e --- /dev/null +++ b/build/pkgs/pyproject-hooks/default.nix @@ -0,0 +1,23 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + flit-core, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pyproject-hooks) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/pyproject-metadata/default.nix b/build/pkgs/pyproject-metadata/default.nix new file mode 100644 index 0000000..9c9ab0d --- /dev/null +++ b/build/pkgs/pyproject-metadata/default.nix @@ -0,0 +1,26 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pyproject-metadata) + pname + version + src + meta + ; + + passthru.dependencies = { + packaging = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/scikit-build/default.nix b/build/pkgs/scikit-build/default.nix new file mode 100644 index 0000000..219e38b --- /dev/null +++ b/build/pkgs/scikit-build/default.nix @@ -0,0 +1,41 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.scikit-build) + pname + version + src + meta + patches + ; + + passthru.dependencies = + { + distro = [ ]; + packaging = [ ]; + setuptools = [ ]; + wheel = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.8") { + typing-extensions = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + hatch-vcs = [ ]; + hatch-fancy-pypi-readme = [ ]; + }; +} diff --git a/build/pkgs/setuptools-git/default.nix b/build/pkgs/setuptools-git/default.nix new file mode 100644 index 0000000..778e7bb --- /dev/null +++ b/build/pkgs/setuptools-git/default.nix @@ -0,0 +1,25 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, + pkgs, +}: +stdenv.mkDerivation { + inherit (python3Packages.setuptools-git) + pname + version + src + meta + ; + + propagatedBuildInputs = [ pkgs.git ]; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/setuptools-scm/default.nix b/build/pkgs/setuptools-scm/default.nix new file mode 100644 index 0000000..a32f861 --- /dev/null +++ b/build/pkgs/setuptools-scm/default.nix @@ -0,0 +1,53 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.setuptools-scm) + pname + version + src + meta + setupHook + ; + + passthru = { + dependencies = + { + packaging = [ ]; + setuptools = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.10") { + typing-extensions = [ ]; + }; + + optional-dependencies = { + toml = { + tomli = [ ]; + }; + rich = { + rich = [ ]; + }; + }; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem ( + { + setuptools = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + } + ); +} diff --git a/build/pkgs/setuptools/default.nix b/build/pkgs/setuptools/default.nix new file mode 100644 index 0000000..520b3a8 --- /dev/null +++ b/build/pkgs/setuptools/default.nix @@ -0,0 +1,24 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.setuptools) + pname + version + src + meta + patches + preBuild # Skips windows files + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/toml/default.nix b/build/pkgs/toml/default.nix new file mode 100644 index 0000000..e25346f --- /dev/null +++ b/build/pkgs/toml/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.toml) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/build/pkgs/tomli/default.nix b/build/pkgs/tomli/default.nix new file mode 100644 index 0000000..b74505d --- /dev/null +++ b/build/pkgs/tomli/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.tomli) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/tomlkit/default.nix b/build/pkgs/tomlkit/default.nix new file mode 100644 index 0000000..ee30db6 --- /dev/null +++ b/build/pkgs/tomlkit/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.tomlkit) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + poetry-core = [ ]; + }; +} diff --git a/build/pkgs/trove-classifiers/default.nix b/build/pkgs/trove-classifiers/default.nix new file mode 100644 index 0000000..46b80eb --- /dev/null +++ b/build/pkgs/trove-classifiers/default.nix @@ -0,0 +1,23 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.trove-classifiers) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + calver = [ ]; + }; +} diff --git a/build/pkgs/typing-extensions/default.nix b/build/pkgs/typing-extensions/default.nix new file mode 100644 index 0000000..5e55fc6 --- /dev/null +++ b/build/pkgs/typing-extensions/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.typing-extensions) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/wheel/default.nix b/build/pkgs/wheel/default.nix new file mode 100644 index 0000000..85c74e2 --- /dev/null +++ b/build/pkgs/wheel/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.wheel) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/build/pkgs/whool/default.nix b/build/pkgs/whool/default.nix new file mode 100644 index 0000000..4e8517a --- /dev/null +++ b/build/pkgs/whool/default.nix @@ -0,0 +1,35 @@ +{ + stdenv, + lib, + python, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.whool) + pname + version + src + meta + setupHook + ; + + passthru.dependencies = + { + manifestoo-core = [ ]; + wheel = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.11") { + tomli = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + hatch-vcs = [ ]; + }; +} diff --git a/build/pkgs/zipp/default.nix b/build/pkgs/zipp/default.nix new file mode 100644 index 0000000..6ccb0d4 --- /dev/null +++ b/build/pkgs/zipp/default.nix @@ -0,0 +1,23 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.zipp) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + setuptools-scm = [ ]; + }; +} diff --git a/build/resolvers.nix b/build/resolvers.nix new file mode 100644 index 0000000..d90d17a --- /dev/null +++ b/build/resolvers.nix @@ -0,0 +1,109 @@ +{ lib }: + +let + inherit (lib) + filterAttrs + concatMap + attrNames + elemAt + genericClosure + match + genAttrs + ; + + # Most of the packages in pkgs/ are build-systems that we can use to create the memo + knownPackages = (attrNames (filterAttrs (_: type: type == "directory") (builtins.readDir ./pkgs))); + +in +{ + /* + Resolve dependencies using a non-circular supporting approach. + + This implementation is faster than the one supporting circular dependencies + resolveNonCyclic is intended to resolve build-system dependencies. + */ + resolveNonCyclic = + # Package set to resolve packages from + set: + let + recurse' = + name: extras: + let + pkg = set.${name}; + dependencies = pkg.passthru.dependencies or { }; + optional-dependencies = pkg.passthru.optional-dependencies or { }; + in + [ name ] + ++ concatMap (name: recurse name dependencies.${name}) (attrNames dependencies) + ++ concatMap ( + extra: + let + extra' = optional-dependencies.${extra}; + in + concatMap (name: recurse name extra'.${name}) (attrNames extra') + ) extras; + + # Memoise known build systems with no extras enabled for better performance + memo = genAttrs knownPackages (name: recurse' name [ ]); + + recurse = + name: extras: if extras == [ ] then (memo.${name} or (recurse' name [ ])) else recurse' name extras; + in + # Attribute set of dependencies -> extras { requests = [ "socks" ]; } + spec: concatMap (name: recurse name spec.${name}) (attrNames spec); + + /* + Resolve dependencies using a cyclic supporting approach. + + resolveCyclic is intended to resolve virtualenv dependencies. + */ + resolveCyclic = + let + mkKey = key: { inherit key; }; + in + # Package set to resolve packages from + set: + # Attribute set of dependencies -> extras { requests = [ "socks" ]; } + spec: + let + # Resolve spec recursively + closure' = genericClosure { + startSet = concatMap ( + name: [ (mkKey name) ] ++ map (extra: mkKey "${name}@${extra}") spec.${name} + ) (attrNames spec); + operator = + { key }: + let + m = match "(.+)@(.*)" key; + in + # We're looking for a package with extra + if m != null then + ( + let + pkg = set.${elemAt m 0}; + dependencies = pkg.passthru.optional-dependencies.${elemAt m 1}; + in + concatMap (name: [ (mkKey name) ] ++ map (extra: mkKey "${name}@${extra}") dependencies.${name}) ( + attrNames dependencies + ) + ) + # Root package with no extra + else + ( + let + pkg = set.${key}; + dependencies = pkg.passthru.dependencies or { }; + in + concatMap (name: [ (mkKey name) ] ++ map (extra: mkKey "${name}@${extra}") dependencies.${name}) ( + attrNames dependencies + ) + ); + }; + + # closure' contains a list like [ { key = "dep@extra"; }] + # where each dependency containing an extra is a duplicate of it's non-extra enabled + closure = lib.filter (dep: match ".+@.*" dep.key == null) closure'; + + in + map (dep: dep.key) closure; +} From 2fe2db982bdf2bb2f2bd56aeb1b89d6709a8a989 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 19 Sep 2024 02:57:38 +0000 Subject: [PATCH 02/22] flake: Add lix-unit devshell --- .envrc | 2 -- .gitignore | 1 + flake.lock | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 21 +++++++++++++++++---- 4 files changed, 70 insertions(+), 6 deletions(-) delete mode 100644 .envrc diff --git a/.envrc b/.envrc deleted file mode 100644 index 8e0fac8..0000000 --- a/.envrc +++ /dev/null @@ -1,2 +0,0 @@ -use flake -watch_file pyproject.toml diff --git a/.gitignore b/.gitignore index bb07b35..42c93af 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ __pypackages__ .direnv .pdm-python +.envrc diff --git a/flake.lock b/flake.lock index 1708710..ad5be22 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,56 @@ { "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "lix-unit", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719745305, + "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "lix-unit": { + "inputs": { + "flake-parts": "flake-parts", + "mdbook-nixdoc": [ + "mdbook-nixdoc" + ], + "nix-github-actions": [ + "nix-github-actions" + ], + "nixpkgs": [ + "nixpkgs" + ], + "treefmt-nix": [ + "treefmt-nix" + ] + }, + "locked": { + "lastModified": 1726522583, + "narHash": "sha256-zRG0mHsdoJKXRxvuRcPTVEDQDg9tYDR8ozcyNWMoxhg=", + "owner": "adisbladis", + "repo": "lix-unit", + "rev": "90e475ec4e71466653a698e1da2a86502be05cfa", + "type": "github" + }, + "original": { + "owner": "adisbladis", + "repo": "lix-unit", + "type": "github" + } + }, "mdbook-nixdoc": { "inputs": { "nix-github-actions": [ @@ -61,6 +112,7 @@ }, "root": { "inputs": { + "lix-unit": "lix-unit", "mdbook-nixdoc": "mdbook-nixdoc", "nix-github-actions": "nix-github-actions", "nixpkgs": "nixpkgs", diff --git a/flake.nix b/flake.nix index 3319749..21b3d69 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,14 @@ mdbook-nixdoc.inputs.nixpkgs.follows = "nixpkgs"; mdbook-nixdoc.inputs.nix-github-actions.follows = "nix-github-actions"; + lix-unit = { + url = "github:adisbladis/lix-unit"; + inputs.mdbook-nixdoc.follows = "mdbook-nixdoc"; + inputs.nix-github-actions.follows = "nix-github-actions"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.treefmt-nix.follows = "treefmt-nix"; + }; + treefmt-nix.url = "github:numtide/treefmt-nix"; treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; }; @@ -19,6 +27,7 @@ nixpkgs, nix-github-actions, treefmt-nix, + lix-unit, ... }@inputs: let @@ -66,17 +75,21 @@ system: let pkgs = nixpkgs.legacyPackages.${system}; - in - { - default = pkgs.mkShell { + mkShell' = { nix-unit }: pkgs.mkShell { packages = [ - pkgs.nix-unit + nix-unit inputs.mdbook-nixdoc.packages.${system}.default (pkgs.python3.withPackages (_ps: [ ])) pkgs.hivemind pkgs.reflex ] ++ self.packages.${system}.doc.nativeBuildInputs; }; + + in + { + nix = mkShell' { inherit (pkgs) nix-unit; }; + lix = mkShell' { nix-unit = lix-unit.packages.${system}.default; }; + default = self.devShells.${system}.nix; } ); From a1a1de3951d41f64fe3da95ad91be6c010ad1ed5 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 19 Sep 2024 02:58:23 +0000 Subject: [PATCH 03/22] build: Add renderers & tests --- build/README.md | 32 ++ build/checks.nix | 27 ++ build/default.nix | 125 +------- build/lib/default.nix | 13 + build/lib/renderers.nix | 52 +++ build/{ => lib}/resolvers.nix | 4 +- build/lib/test.nix | 9 + build/lib/test_renderers.nix | 302 ++++++++++++++++++ build/packages.nix | 144 +++++++++ build/pkgs/cffi/default.nix | 8 +- build/pkgs/hatch-vcs/default.nix | 14 +- build/pkgs/meson/default.nix | 18 ++ build/pkgs/numpy/default.nix | 35 ++ build/pkgs/oldest-supported-numpy/default.nix | 28 ++ build/pkgs/pytest-runner/default.nix | 23 ++ build/pkgs/setuptools/default.nix | 2 + build/pkgs/versioneer/default.nix | 26 ++ flake.nix | 51 ++- lib/renderers.nix | 74 +++-- lib/test_project.nix | 1 + lib/test_renderers.nix | 8 + 21 files changed, 819 insertions(+), 177 deletions(-) create mode 100644 build/README.md create mode 100644 build/checks.nix create mode 100644 build/lib/default.nix create mode 100644 build/lib/renderers.nix rename build/{ => lib}/resolvers.nix (96%) create mode 100644 build/lib/test.nix create mode 100644 build/lib/test_renderers.nix create mode 100644 build/packages.nix create mode 100644 build/pkgs/numpy/default.nix create mode 100644 build/pkgs/oldest-supported-numpy/default.nix create mode 100644 build/pkgs/pytest-runner/default.nix create mode 100644 build/pkgs/versioneer/default.nix diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000..340aa30 --- /dev/null +++ b/build/README.md @@ -0,0 +1,32 @@ +# Pyproject.nix build infrastructure + +Pyproject.nix can be used with nixpkgs `buildPythonPackage`/`packageOverrides`/`withPackages`, but also implements it's own build infrastructure. +It is experimental, and mainly targeted at expert users such as 2nix authors. If you're looking for packaging guidance you're in the wrong place. + +This fixes many problems existing in the Nixpkgs Python infrastructure such as: + + No dependency propagation + +No leaking of dependencies via PYTHONPATH + + No runtime deps at build time + +This causes much less rebuilds + + Support for circular dependencies + + No wrapping of Python bin's + +No strange venv breaking shell wrapper + + Proper virtualenvs only + +Meaning that applications that spawns children using sys.executable works fine. + + Hermetic bootstrap + +Nixpkgs breaks if you override any bootstrap dependencies with wheels. + + Manual cross splicing + +Meaning you can override build-time and runtime deps separately. diff --git a/build/checks.nix b/build/checks.nix new file mode 100644 index 0000000..72f500a --- /dev/null +++ b/build/checks.nix @@ -0,0 +1,27 @@ +{ pyproject-nix, pkgs }: + +let + python = pkgs.python312; + + # Inject your own packages on top with overrideScope + pythonSet = pkgs.callPackage pyproject-nix.build.packages { + inherit python; + }; + + testVenv = pythonSet.pythonPackagesHostHost.mkVirtualEnv "test-venv" { + build = [ ]; + }; + +in + +{ + make-venv = + pkgs.runCommand "venv-run-build-test" + { + nativeBuildInputs = [ testVenv ]; + } + '' + pyproject-build --help > /dev/null + touch $out + ''; +} diff --git a/build/default.nix b/build/default.nix index 7894eae..e007ecc 100644 --- a/build/default.nix +++ b/build/default.nix @@ -1,116 +1,9 @@ -{ - lib, - python, - newScope, - buildPackages, - stdenv, -}: - -let - inherit (lib) makeScope; - - resolvers = import ./resolvers.nix { inherit lib; }; - inherit (resolvers) resolveCyclic resolveNonCyclic; - - mkResolveBuildSystem = - set: - let - resolveNonCyclic' = resolveNonCyclic set; - in - spec: map (name: set.${name}) (resolveNonCyclic' spec); - - mkResolveVirtualEnv = set: spec: map (name: set.${name}) (resolveCyclic set spec); - - pkgsFun = - let - mkPkgs = import ./pkgs { inherit lib; }; - in - final: mkPkgs { inherit (final) callPackage pyprojectBootstrapHook; }; - -in -makeScope newScope ( - final: - let - mkPythonSet = - { - newScope, - python, - stdenv, - pythonPackagesBuildHost, - bootstrapHooks, - pythonPackagesFun, - }: - makeScope newScope ( - pkgsFinal: - { - inherit python stdenv pythonPackagesBuildHost; - - # Pyproject hook used for bootstrap packages - pyprojectBootstrapHook = pkgsFinal.pyprojectHook.override { - inherit (bootstrapHooks) pyprojectConfigureHook pyprojectBuildHook; - }; - - # Initialize dependency resolvers - resolveBuildSystem = mkResolveBuildSystem pythonPackagesBuildHost; - resolveVirtualEnv = mkResolveVirtualEnv pkgsFinal; - - hooks = pkgsFinal.callPackage ./hooks { }; - inherit (pkgsFinal.hooks) - pyprojectConfigureHook - pyprojectBuildHook - pyprojectInstallHook - pyprojectBytecodeHook - pyprojectOutputSetupHook - pyprojectHook - ; - } - // pythonPackagesFun pkgsFinal - ); - - bootstrapHooks = final.callPackage ./hooks { - python = final.python.pythonOnBuildForHost; - resolveBuildSystem = mkResolveBuildSystem final.pythonPackagesBootstrap; - hooks = bootstrapHooks; - }; - - in - { - # Allows overriding Python by calling overrideScope on the outer scope - inherit python; - - pythonPackagesBootstrap = mkPythonSet { - inherit (buildPackages) stdenv newScope; - inherit bootstrapHooks; - python = python.pythonOnBuildForHost; - pythonPackagesBuildHost = final.pythonPackagesBootstrap; - pythonPackagesFun = - _: - final.callPackage ./bootstrap.nix { - inherit (bootstrapHooks) pyprojectInstallHook pyprojectBytecodeHook pyprojectOutputSetupHook; - python = final.python.pythonOnBuildForHost; - }; - }; - - # Python packages for the build host - pythonPackagesBuildHost = mkPythonSet { - inherit (buildPackages) stdenv newScope; - python = python.pythonOnBuildForHost; - inherit (final) pythonPackagesBuildHost; - bootstrapHooks = final.pythonPackagesBootstrap.hooks; - pythonPackagesFun = pkgsFun; - }; - - # Python packages for the target host - pythonPackagesHostHost = - # If we're not doing cross reference build host packages - if stdenv.buildPlatform != stdenv.hostPlatform then - mkPythonSet { - inherit (final) newScope pythonPackagesBuildHost; - inherit python stdenv; - bootstrapHooks = final.pythonPackagesBuildHost.hooks; - pythonPackagesFun = pkgsFun; - } - else - final.pythonPackagesBuildHost; - } -) +{ lib, pyproject-nix }: + +lib.fix (self: { + packages = import ./packages.nix { + inherit (self.lib) resolvers; + inherit lib; + }; + lib = import ./lib { inherit lib pyproject-nix; }; +}) diff --git a/build/lib/default.nix b/build/lib/default.nix new file mode 100644 index 0000000..f6da70f --- /dev/null +++ b/build/lib/default.nix @@ -0,0 +1,13 @@ +{ lib, pyproject-nix }: +let + inherit (builtins) mapAttrs; + inherit (lib) fix; +in + +fix ( + self: + mapAttrs (_: path: import path ({ inherit pyproject-nix lib; } // self)) { + renderers = ./renderers.nix; + resolvers = ./resolvers.nix; + } +) diff --git a/build/lib/renderers.nix b/build/lib/renderers.nix new file mode 100644 index 0000000..313b1fb --- /dev/null +++ b/build/lib/renderers.nix @@ -0,0 +1,52 @@ +{ lib, pyproject-nix, ... }: + +let + inherit (lib) + optionalAttrs + mapAttrs + concatMap + groupBy + ; + inherit (pyproject-nix.lib) pep621; + + # Make a dependency specification attrset from a list of dependencies + mkSpec = + dependencies: mapAttrs (_: concatMap (dep: dep.extras)) (groupBy (dep: dep.name) dependencies); + +in +{ + + mkDerivation = + { + project, + environ, + extras ? [ ], + }: + let + inherit (project) pyproject; + + filteredDeps = pep621.filterDependenciesByEnviron environ extras project.dependencies; + + in + { pyprojectHook, resolveBuildSystem }: + { + passthru = { + dependencies = mkSpec filteredDeps.dependencies; + optional-dependencies = mapAttrs (_: mkSpec) filteredDeps.extras; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem (mkSpec filteredDeps.build-systems); + + meta = pyproject-nix.lib.renderers.meta { + inherit project; + }; + } + // optionalAttrs (pyproject.project ? name) { pname = pyproject.project.name; } + // optionalAttrs (pyproject.project ? version) { inherit (pyproject.project) version; } + // optionalAttrs (!pyproject.project ? version && pyproject.project ? name) { + inherit (pyproject.project) name; + } + // optionalAttrs (project.projectRoot != null) { src = project.projectRoot; }; +} diff --git a/build/resolvers.nix b/build/lib/resolvers.nix similarity index 96% rename from build/resolvers.nix rename to build/lib/resolvers.nix index d90d17a..8161903 100644 --- a/build/resolvers.nix +++ b/build/lib/resolvers.nix @@ -1,4 +1,4 @@ -{ lib }: +{ lib, ... }: let inherit (lib) @@ -12,7 +12,7 @@ let ; # Most of the packages in pkgs/ are build-systems that we can use to create the memo - knownPackages = (attrNames (filterAttrs (_: type: type == "directory") (builtins.readDir ./pkgs))); + knownPackages = attrNames (filterAttrs (_: type: type == "directory") (builtins.readDir ../pkgs)); in { diff --git a/build/lib/test.nix b/build/lib/test.nix new file mode 100644 index 0000000..ea2a309 --- /dev/null +++ b/build/lib/test.nix @@ -0,0 +1,9 @@ +{ + pyproject-nix, + lib, + pkgs, +}: + +{ + renderers = import ./test_renderers.nix { inherit pkgs lib pyproject-nix; }; +} diff --git a/build/lib/test_renderers.nix b/build/lib/test_renderers.nix new file mode 100644 index 0000000..c2720ee --- /dev/null +++ b/build/lib/test_renderers.nix @@ -0,0 +1,302 @@ +{ + pyproject-nix, + lib, + pkgs, +}: + +let + inherit (pyproject-nix.build.lib.renderers) mkDerivation; + inherit (pyproject-nix.lib.project) loadPyproject; + + libFixtures = import ../../lib/fixtures; + + python = pkgs.python312; + + pythonSet = pkgs.callPackage pyproject-nix.build.packages { inherit python; }; + +in + +{ + mkDerivation = + let + environ = pyproject-nix.lib.pep508.mkEnviron pkgs.python312; + + renderFixture = + fixture: + let + rendered = + (mkDerivation { + project = loadPyproject { + pyproject = libFixtures.${fixture}; + }; + inherit environ; + }) + { + pyprojectHook = null; + inherit (pythonSet.pythonPackagesHostHost) resolveBuildSystem; + }; + in + rendered + // { + nativeBuildInputs = map ( + input: if input == null then null else input.pname + ) rendered.nativeBuildInputs; + }; + + in + { + testPandas = { + expr = renderFixture "pandas.toml"; + expected = { + meta = { + description = "Powerful data structures for data analysis, time series, and statistics"; + }; + nativeBuildInputs = [ + null + "cython" + "meson" + "ninja" + "meson-python" + "meson" + "ninja" + "pyproject-metadata" + "packaging" + "tomli" + "oldest-supported-numpy" + "numpy" + "versioneer" + "tomli" + "wheel" + ]; + passthru = { + dependencies = { + numpy = [ ]; + python-dateutil = [ ]; + pytz = [ ]; + tzdata = [ ]; + }; + optional-dependencies = { + all = { + beautifulsoup4 = [ ]; + bottleneck = [ ]; + brotlipy = [ ]; + fastparquet = [ ]; + fsspec = [ ]; + gcsfs = [ ]; + html5lib = [ ]; + hypothesis = [ ]; + jinja2 = [ ]; + lxml = [ ]; + matplotlib = [ ]; + numba = [ ]; + numexpr = [ ]; + odfpy = [ ]; + openpyxl = [ ]; + pandas-gbq = [ ]; + psycopg2 = [ ]; + pyarrow = [ ]; + pymysql = [ ]; + pyqt5 = [ ]; + pyreadstat = [ ]; + pytest = [ ]; + pytest-asyncio = [ ]; + pytest-xdist = [ ]; + python-snappy = [ ]; + pyxlsb = [ ]; + qtpy = [ ]; + s3fs = [ ]; + scipy = [ ]; + sqlalchemy = [ ]; + tables = [ ]; + tabulate = [ ]; + xarray = [ ]; + xlrd = [ ]; + xlsxwriter = [ ]; + zstandard = [ ]; + }; + aws = { + s3fs = [ ]; + }; + clipboard = { + pyqt5 = [ ]; + qtpy = [ ]; + }; + compression = { + brotlipy = [ ]; + python-snappy = [ ]; + zstandard = [ ]; + }; + computation = { + scipy = [ ]; + xarray = [ ]; + }; + excel = { + odfpy = [ ]; + openpyxl = [ ]; + pyxlsb = [ ]; + xlrd = [ ]; + xlsxwriter = [ ]; + }; + feather = { + pyarrow = [ ]; + }; + fss = { + fsspec = [ ]; + }; + gcp = { + gcsfs = [ ]; + pandas-gbq = [ ]; + }; + hdf5 = { + tables = [ ]; + }; + html = { + beautifulsoup4 = [ ]; + html5lib = [ ]; + lxml = [ ]; + }; + mysql = { + pymysql = [ ]; + sqlalchemy = [ ]; + }; + output_formatting = { + jinja2 = [ ]; + tabulate = [ ]; + }; + parquet = { + pyarrow = [ ]; + }; + performance = { + bottleneck = [ ]; + numba = [ ]; + numexpr = [ ]; + }; + plot = { + matplotlib = [ ]; + }; + postgresql = { + psycopg2 = [ ]; + sqlalchemy = [ ]; + }; + spss = { + pyreadstat = [ ]; + }; + sql-other = { + sqlalchemy = [ ]; + }; + test = { + hypothesis = [ ]; + pytest = [ ]; + pytest-asyncio = [ ]; + pytest-xdist = [ ]; + }; + xml = { + lxml = [ ]; + }; + }; + }; + pname = "pandas"; + name = "pandas"; + }; + }; + + testPdm = { + expr = renderFixture "pdm.toml"; + expected = { + meta = { + description = "A modern Python package and dependency manager supporting the latest PEP standards"; + license = { + deprecated = false; + free = true; + fullName = "MIT License"; + redistributable = true; + shortName = "mit"; + spdxId = "MIT"; + url = "https://spdx.org/licenses/MIT.html"; + }; + mainProgram = "pdm"; + }; + nativeBuildInputs = [ + null + "pdm-backend" + ]; + passthru = { + dependencies = { + blinker = [ ]; + cachecontrol = [ "filecache" ]; + certifi = [ ]; + findpython = [ ]; + installer = [ ]; + packaging = [ ]; + platformdirs = [ ]; + pyproject-hooks = [ ]; + python-dotenv = [ ]; + requests-toolbelt = [ ]; + resolvelib = [ ]; + rich = [ ]; + shellingham = [ ]; + tomlkit = [ ]; + unearth = [ ]; + virtualenv = [ ]; + }; + optional-dependencies = { + all = { + pdm = [ + "keyring" + "template" + "truststore" + ]; + }; + cookiecutter = { + cookiecutter = [ ]; + }; + copier = { + copier = [ ]; + }; + keyring = { + keyring = [ ]; + }; + pytest = { + pytest = [ ]; + pytest-mock = [ ]; + }; + template = { + pdm = [ + "copier" + "cookiecutter" + ]; + }; + truststore = { + truststore = [ ]; + }; + }; + }; + pname = "pdm"; + name = "pdm"; + }; + }; + + testUv = { + expr = renderFixture "uv.toml"; + expected = { + meta = { + description = "Add your description here"; + }; + nativeBuildInputs = [ + null + "hatchling" + "packaging" + "pathspec" + "pluggy" + "trove-classifiers" + ]; + passthru = { + dependencies = { }; + optional-dependencies = { }; + }; + pname = "uv-fixture"; + version = "0.1.0"; + }; + }; + }; +} diff --git a/build/packages.nix b/build/packages.nix new file mode 100644 index 0000000..c6dedfa --- /dev/null +++ b/build/packages.nix @@ -0,0 +1,144 @@ +{ lib, resolvers }: +let + inherit (resolvers) resolveCyclic resolveNonCyclic; + inherit (lib) makeScope; + + mkResolveBuildSystem = + set: + let + resolveNonCyclic' = resolveNonCyclic set; + in + spec: + map (name: set.${name}) ( + resolveNonCyclic' ( + if spec != { } then + spec + else + { + # Implement fallback behaviour in case of empty build-system + setuptools = [ ]; + wheel = [ ]; + } + ) + ); + + mkResolveVirtualEnv = set: spec: map (name: set.${name}) (resolveCyclic set spec); + + pkgsFun = + let + mkPkgs = import ./pkgs { inherit lib; }; + in + final: mkPkgs { inherit (final) callPackage pyprojectBootstrapHook; }; + +in + +{ + python, + newScope, + buildPackages, + stdenv, +}: +makeScope newScope ( + final: + let + mkPythonSet = + { + newScope, + python, + stdenv, + pythonPackagesBuildHost, + bootstrapHooks, + pythonPackagesFun, + }: + makeScope newScope ( + pkgsFinal: + { + inherit python stdenv pythonPackagesBuildHost; + + # Pyproject hook used for bootstrap packages + pyprojectBootstrapHook = pkgsFinal.pyprojectHook.override { + inherit (bootstrapHooks) pyprojectConfigureHook pyprojectBuildHook; + }; + + # Initialize dependency resolvers + resolveBuildSystem = mkResolveBuildSystem pythonPackagesBuildHost; + resolveVirtualEnv = mkResolveVirtualEnv pkgsFinal; + + # Make a virtual env from resolved dependencies + mkVirtualEnv = + name: spec: + pkgsFinal.stdenv.mkDerivation { + inherit name; + + dontConfigure = true; + dontUnpack = true; + dontBuild = true; + + nativeBuildInputs = [ + pkgsFinal.pyprojectMakeVenvHook + ]; + + buildInputs = pkgsFinal.resolveVirtualEnv spec; + }; + + hooks = pkgsFinal.callPackage ./hooks { }; + inherit (pkgsFinal.hooks) + pyprojectConfigureHook + pyprojectBuildHook + pyprojectInstallHook + pyprojectBytecodeHook + pyprojectOutputSetupHook + pyprojectMakeVenvHook + pyprojectHook + ; + } + // pythonPackagesFun pkgsFinal + ); + + bootstrapHooks = final.callPackage ./hooks { + python = final.python.pythonOnBuildForHost; + resolveBuildSystem = mkResolveBuildSystem final.pythonPackagesBootstrap; + hooks = bootstrapHooks; + }; + + in + { + # Allows overriding Python by calling overrideScope on the outer scope + inherit python; + + pythonPackagesBootstrap = mkPythonSet { + inherit (buildPackages) stdenv newScope; + inherit bootstrapHooks; + python = python.pythonOnBuildForHost; + pythonPackagesBuildHost = final.pythonPackagesBootstrap; + pythonPackagesFun = + _: + final.callPackage ./bootstrap.nix { + inherit (bootstrapHooks) pyprojectInstallHook pyprojectBytecodeHook pyprojectOutputSetupHook; + python = final.python.pythonOnBuildForHost; + }; + }; + + # Python packages for the build host + pythonPackagesBuildHost = mkPythonSet { + inherit (buildPackages) stdenv newScope; + python = python.pythonOnBuildForHost; + inherit (final) pythonPackagesBuildHost; + bootstrapHooks = final.pythonPackagesBootstrap.hooks; + pythonPackagesFun = pkgsFun; + }; + + # Python packages for the target host + pythonPackagesHostHost = + # If we're not doing cross reference build host packages + if stdenv.buildPlatform != stdenv.hostPlatform then + mkPythonSet { + inherit (final) newScope pythonPackagesBuildHost; + inherit python stdenv; + bootstrapHooks = final.pythonPackagesBuildHost.hooks; + pythonPackagesFun = pkgsFun; + } + else + final.pythonPackagesBuildHost; + } +) diff --git a/build/pkgs/cffi/default.nix b/build/pkgs/cffi/default.nix index e8c2e8f..691753a 100644 --- a/build/pkgs/cffi/default.nix +++ b/build/pkgs/cffi/default.nix @@ -1,6 +1,5 @@ { stdenv, - lib, python, python3Packages, pyprojectHook, @@ -18,10 +17,9 @@ stdenv.mkDerivation { postPatch ; - env = - { - inherit (python3Packages.cffi) NIX_CFLAGS_COMPILE; - }; + env = { + inherit (python3Packages.cffi) NIX_CFLAGS_COMPILE; + }; buildInputs = [ libffi ]; diff --git a/build/pkgs/hatch-vcs/default.nix b/build/pkgs/hatch-vcs/default.nix index 132c12f..f9560f7 100644 --- a/build/pkgs/hatch-vcs/default.nix +++ b/build/pkgs/hatch-vcs/default.nix @@ -17,11 +17,11 @@ stdenv.mkDerivation (_finalAttrs: { setuptools-scm = [ ]; }; - nativeBuildInputs = [ - pyprojectHook - ]; - - build_system = resolveBuildSystem { - hatchling = [ ]; - }; + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + }; }) diff --git a/build/pkgs/meson/default.nix b/build/pkgs/meson/default.nix index a7e0018..addc474 100644 --- a/build/pkgs/meson/default.nix +++ b/build/pkgs/meson/default.nix @@ -3,6 +3,8 @@ pyprojectHook, resolveBuildSystem, pkgs, + lib, + python, }: stdenv.mkDerivation { inherit (pkgs.meson) @@ -14,6 +16,22 @@ stdenv.mkDerivation { setupHook ; + passthru.optional-dependencies = { + ninja = { + ninja = [ ]; + }; + progess = { + tqdm = [ ]; + }; + typing = + { + mypy = [ ]; + } + // lib.optionalAttrs (python.pythonOlder "3.8") { + typing-extensions = [ ]; + }; + }; + nativeBuildInputs = [ pyprojectHook diff --git a/build/pkgs/numpy/default.nix b/build/pkgs/numpy/default.nix new file mode 100644 index 0000000..2c5fa6c --- /dev/null +++ b/build/pkgs/numpy/default.nix @@ -0,0 +1,35 @@ +{ + stdenv, + pyprojectHook, + resolveBuildSystem, + python3Packages, + gfortran, + pkg-config, +}: +stdenv.mkDerivation { + inherit (python3Packages.numpy) + pname + version + src + meta + patches + postPatch + preConfigure + postConfigure + preBuild + buildInputs + enableParallelBuilding + passthru + ; + + nativeBuildInputs = + [ + pyprojectHook + gfortran + pkg-config + ] + ++ resolveBuildSystem { + meson-python = [ ]; + cython = [ ]; + }; +} diff --git a/build/pkgs/oldest-supported-numpy/default.nix b/build/pkgs/oldest-supported-numpy/default.nix new file mode 100644 index 0000000..293b9e4 --- /dev/null +++ b/build/pkgs/oldest-supported-numpy/default.nix @@ -0,0 +1,28 @@ +{ + stdenv, + pyprojectHook, + resolveBuildSystem, + python3Packages, +}: +stdenv.mkDerivation { + inherit (python3Packages.oldest-supported-numpy) + pname + version + src + meta + postPatch + ; + + passthru.dependencies = { + numpy = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + wheel = [ ]; + }; +} diff --git a/build/pkgs/pytest-runner/default.nix b/build/pkgs/pytest-runner/default.nix new file mode 100644 index 0000000..1251ad7 --- /dev/null +++ b/build/pkgs/pytest-runner/default.nix @@ -0,0 +1,23 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.pytest-runner) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + setuptools-scm = [ ]; + }; +} diff --git a/build/pkgs/setuptools/default.nix b/build/pkgs/setuptools/default.nix index 520b3a8..013eb76 100644 --- a/build/pkgs/setuptools/default.nix +++ b/build/pkgs/setuptools/default.nix @@ -14,6 +14,8 @@ stdenv.mkDerivation { preBuild # Skips windows files ; + passthru.dependencies.wheel = [ ]; + nativeBuildInputs = [ pyprojectHook diff --git a/build/pkgs/versioneer/default.nix b/build/pkgs/versioneer/default.nix new file mode 100644 index 0000000..6d1b73d --- /dev/null +++ b/build/pkgs/versioneer/default.nix @@ -0,0 +1,26 @@ +{ + stdenv, + pyprojectHook, + resolveBuildSystem, + python3Packages, +}: +stdenv.mkDerivation { + inherit (python3Packages.versioneer) + pname + version + src + meta + ; + + passthru.optional-dependencies.toml = { + tomli = [ ]; + }; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + setuptools = [ ]; + }; +} diff --git a/flake.nix b/flake.nix index 21b3d69..9baf228 100644 --- a/flake.nix +++ b/flake.nix @@ -43,6 +43,12 @@ }; }; + # Note: This build infrastructure is experimental. + build = import ./build { + pyproject-nix = self; + inherit lib; + }; + lib = import ./lib { inherit lib; }; templates = @@ -65,25 +71,35 @@ ); # Expose unit tests for external discovery - libTests = import ./lib/test.nix { - inherit lib; - pyproject = self.lib; - pkgs = nixpkgs.legacyPackages.x86_64-linux; - }; + libTests = + import ./lib/test.nix { + inherit lib; + pyproject = self.lib; + pkgs = nixpkgs.legacyPackages.x86_64-linux; + } + // { + build = import ./build/lib/test.nix { + pyproject-nix = self; + inherit lib; + pkgs = nixpkgs.legacyPackages.x86_64-linux; + }; + }; devShells = forAllSystems ( system: let pkgs = nixpkgs.legacyPackages.${system}; - mkShell' = { nix-unit }: pkgs.mkShell { - packages = [ - nix-unit - inputs.mdbook-nixdoc.packages.${system}.default - (pkgs.python3.withPackages (_ps: [ ])) - pkgs.hivemind - pkgs.reflex - ] ++ self.packages.${system}.doc.nativeBuildInputs; - }; + mkShell' = + { nix-unit }: + pkgs.mkShell { + packages = [ + nix-unit + inputs.mdbook-nixdoc.packages.${system}.default + (pkgs.python3.withPackages (_ps: [ ])) + pkgs.hivemind + pkgs.reflex + ] ++ self.packages.${system}.doc.nativeBuildInputs; + }; in { @@ -98,7 +114,12 @@ let pkgs = nixpkgs.legacyPackages.${system}; in - pkgs.callPackages ./test { pyproject = import ./default.nix { inherit lib; }; } + (lib.mapAttrs' (name: drv: lib.nameValuePair "nixpkgs-${name}" drv) ( + pkgs.callPackages ./test { pyproject = import ./default.nix { inherit lib; }; } + )) + // (lib.mapAttrs' (name: drv: lib.nameValuePair "build-${name}" drv) ( + pkgs.callPackages ./build/checks.nix { pyproject-nix = self; } + )) // { formatter = pkgs.writeShellScript "fmt-check" '' set -euo pipefail diff --git a/lib/renderers.nix b/lib/renderers.nix index 88dfee9..6e46365 100644 --- a/lib/renderers.nix +++ b/lib/renderers.nix @@ -130,37 +130,6 @@ in inherit (project) pyproject; - meta = - let - project' = project.pyproject.project; - urls = project'.urls or { }; - in - # Optional changelog - optionalAttrs (urls ? changelog) { inherit (urls) changelog; } - // - # Optional description - optionalAttrs (project' ? description) { inherit (project') description; } - // - # Optional license - optionalAttrs (project' ? license.text) ( - assert !(project'.license ? file); - { - # From PEP-621: - # "The text key has a string value which is the license of the project whose meaning is that of the License field from the core metadata. - # These keys are mutually exclusive, so a tool MUST raise an error if the metadata specifies both keys." - # Hence the assert above. - license = licensesBySpdxId.${project'.license.text} or project'.license.text; - } - ) - // - # Only set mainProgram if we only have one script, otherwise it's ambigious which one is main - ( - let - scriptNames = attrNames project'.scripts; - in - optionalAttrs (project' ? scripts && length scriptNames == 1) { mainProgram = head scriptNames; } - ); - optional-dependencies = lib.mapAttrs (_group: getDependencies) project.dependencies.extras; in @@ -181,7 +150,8 @@ in dependencies = getDependencies filteredDeps.dependencies ++ concatMap (group: optional-dependencies.${group} or [ ]) extras; - inherit optional-dependencies meta; + inherit optional-dependencies; + meta = renderers.meta { inherit project; }; } // optionalAttrs (format != "pyproject") { inherit format; } // optionalAttrs (format != "wheel") { @@ -277,4 +247,44 @@ in map (attr: nameValuePair attr attrs.${attr}) (attrValues args.extrasAttrMappings) ); }; + + /* + Renders a project as a meta attribute + + This is used internally in renderers.mkPythonPackages + + Type: meta :: AttrSet -> AttrSet + */ + meta = + { project }: + let + project' = project.pyproject.project; + urls = project'.urls or { }; + in + # Optional changelog + optionalAttrs (urls ? changelog) { inherit (urls) changelog; } + // + # Optional description + optionalAttrs (project' ? description) { inherit (project') description; } + // + # Optional license + optionalAttrs (project' ? license.text) ( + assert !(project'.license ? file); + { + # From PEP-621: + # "The text key has a string value which is the license of the project whose meaning is that of the License field from the core metadata. + # These keys are mutually exclusive, so a tool MUST raise an error if the metadata specifies both keys." + # Hence the assert above. + license = licensesBySpdxId.${project'.license.text} or project'.license.text; + } + ) + // + # Only set mainProgram if we only have one script, otherwise it's ambigious which one is main + ( + let + scriptNames = attrNames project'.scripts; + in + optionalAttrs (project' ? scripts && length scriptNames == 1) { mainProgram = head scriptNames; } + ); + } diff --git a/lib/test_project.nix b/lib/test_project.nix index 118c0f6..571e18c 100644 --- a/lib/test_project.nix +++ b/lib/test_project.nix @@ -391,6 +391,7 @@ lib.fix (self: { expected = { renderers = [ "buildPythonPackage" + "meta" "mkPythonEditablePackage" "withPackages" ]; diff --git a/lib/test_renderers.nix b/lib/test_renderers.nix index e2a072c..94c7ffb 100644 --- a/lib/test_renderers.nix +++ b/lib/test_renderers.nix @@ -68,6 +68,14 @@ rec { }; }; + # Implicitly tested by buildPythonPackage test + meta = { + testDummy = { + expr = null; + expected = null; + }; + }; + buildPythonPackage = { testPdm = { expr = clearDrvInputs ( From b75f87be64d76c9a2116d51a23851e408417bacb Mon Sep 17 00:00:00 2001 From: adisbladis Date: Thu, 19 Sep 2024 04:24:10 +0000 Subject: [PATCH 04/22] build: Add prebuilt wheel support --- build/checks.nix | 19 +++++++++++++++++++ build/hooks/default.nix | 11 +++++++++++ build/hooks/pyproject-wheel-dist-hook.sh | 20 ++++++++++++++++++++ build/packages.nix | 1 + 4 files changed, 51 insertions(+) create mode 100644 build/hooks/pyproject-wheel-dist-hook.sh diff --git a/build/checks.nix b/build/checks.nix index 72f500a..e2a545c 100644 --- a/build/checks.nix +++ b/build/checks.nix @@ -24,4 +24,23 @@ in pyproject-build --help > /dev/null touch $out ''; + + prebuilt-wheel = pythonSet.pythonPackagesHostHost.callPackage ( + { + stdenv, + fetchurl, + pyprojectWheelHook, + }: + stdenv.mkDerivation { + pname = "arpeggio"; + version = "2.0.2"; + + src = fetchurl { + url = "https://files.pythonhosted.org/packages/f7/4f/d28bf30a19d4649b40b501d531b44e73afada99044df100380fd9567e92f/Arpeggio-2.0.2-py2.py3-none-any.whl"; + hash = "sha256-98iuT0BWqJ4CDCTHICrI3z4ryE5BZ0byCw2jW7HeAlA="; + }; + + nativeBuildInputs = [ pyprojectWheelHook ]; + } + ) { }; } diff --git a/build/hooks/default.nix b/build/hooks/default.nix index b93e0b2..235d0ab 100644 --- a/build/hooks/default.nix +++ b/build/hooks/default.nix @@ -51,6 +51,13 @@ in } ./pyproject-build-hook.sh ) { }; + pyprojectWheelDistHook = callPackage ( + _: + makeSetupHook { + name = "pyproject-wheel-dist-hook"; + } ./pyproject-wheel-dist-hook.sh + ) { }; + pyprojectInstallHook = callPackage ( @@ -149,4 +156,8 @@ in pyprojectInstallHook = hooks.pyprojectPypaInstallHook; }) ); + + pyprojectWheelHook = hooks.pyprojectHook.override { + pyprojectBuildHook = hooks.pyprojectWheelDistHook; + }; } diff --git a/build/hooks/pyproject-wheel-dist-hook.sh b/build/hooks/pyproject-wheel-dist-hook.sh new file mode 100644 index 0000000..932b193 --- /dev/null +++ b/build/hooks/pyproject-wheel-dist-hook.sh @@ -0,0 +1,20 @@ +# Setup hook to move a prebuilt wheel into dist as expected by install hook +echo "Sourcing pyproject-wheel-dist-hook" + +pyprojectWheelDist() { + echo "Executing pyprojectWheelDist" + runHook preBuild + + echo "Creating dist..." + mkdir -p dist + ln -s "$src" "dist/$(stripHash "$src")" + + runHook postBuild + echo "Finished executing pyprojectWheelDist" +} + +if [ -z "${dontUsePyprojectWheelDist-}" ] && [ -z "${buildPhase-}" ]; then + echo "Using pyprojectWheelDist" + buildPhase=pyprojectWheelDist + dontUnpack=1 +fi diff --git a/build/packages.nix b/build/packages.nix index c6dedfa..2237237 100644 --- a/build/packages.nix +++ b/build/packages.nix @@ -90,6 +90,7 @@ makeScope newScope ( pyprojectOutputSetupHook pyprojectMakeVenvHook pyprojectHook + pyprojectWheelHook ; } // pythonPackagesFun pkgsFinal From a25825ac6046c6ee6b73dc54063ab829573503e4 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Sep 2024 01:56:55 +0000 Subject: [PATCH 05/22] build: Add test for overriden bootstrap dependency --- build/checks.nix | 37 +++++++++++++++++++++++++++++++++++++ build/lib/default.nix | 11 ++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/build/checks.nix b/build/checks.nix index e2a545c..385a76b 100644 --- a/build/checks.nix +++ b/build/checks.nix @@ -1,6 +1,8 @@ { pyproject-nix, pkgs }: let + inherit (pyproject-nix.build.lib) isBootstrapPackage; + python = pkgs.python312; # Inject your own packages on top with overrideScope @@ -43,4 +45,39 @@ in nativeBuildInputs = [ pyprojectWheelHook ]; } ) { }; + + # Bootstrap dependencies need to pyprojectBootstrapHook + overriden-bootstrap-dep = + let + overlay = final: _prev: { + packaging = final.stdenv.mkDerivation { + pname = "packaging"; + version = "24.1"; + + src = pkgs.fetchurl { + url = "https://files.pythonhosted.org/packages/51/65/50db4dda066951078f0a96cf12f4b9ada6e4b811516bf0262c0f4f7064d4/packaging-24.1.tar.gz"; + hash = "sha256-Am7XLI7T/M5b+JUFciWGmJJ/0dvaEKXpgc3wrDf08AI="; + }; + + nativeBuildInputs = + assert isBootstrapPackage "packaging"; + [ + final.pyprojectBootstrapHook + ] + ++ final.resolveBuildSystem { + flit-core = [ ]; + }; + }; + }; + + pythonSet' = pythonSet.overrideScope ( + _final: prev: { + pythonPackagesBuildHost = prev.pythonPackagesBuildHost.overrideScope overlay; + } + ); + + in + pythonSet'.pythonPackagesHostHost.mkVirtualEnv "overriden-bootstrap-venv" { + build = [ ]; + }; } diff --git a/build/lib/default.nix b/build/lib/default.nix index f6da70f..447867e 100644 --- a/build/lib/default.nix +++ b/build/lib/default.nix @@ -1,7 +1,7 @@ { lib, pyproject-nix }: let inherit (builtins) mapAttrs; - inherit (lib) fix; + inherit (lib) fix elem flip; in fix ( @@ -10,4 +10,13 @@ fix ( renderers = ./renderers.nix; resolvers = ./resolvers.nix; } + // { + + isBootstrapPackage = flip elem [ + "flit-core" + "pyproject-hooks" + "packaging" + "build" + ]; + } ) From e3b50e02543fe6faef9b7e93af8ff4de5e778611 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Sep 2024 05:08:31 +0000 Subject: [PATCH 06/22] build: Add doc --- build/README.md | 32 ------------ build/bootstrap.nix | 1 - build/checks.nix | 2 +- build/lib/renderers.nix | 10 ++++ build/lib/resolvers.nix | 22 ++++---- build/packages.nix | 17 +++--- doc/src/SUMMARY.md | 12 +++-- doc/src/build | 1 + doc/src/build.md | 108 +++++++++++++++++++++++++++++++++++++++ doc/src/nixpkgs-build.md | 3 ++ 10 files changed, 153 insertions(+), 55 deletions(-) delete mode 100644 build/README.md create mode 120000 doc/src/build create mode 100644 doc/src/build.md create mode 100644 doc/src/nixpkgs-build.md diff --git a/build/README.md b/build/README.md deleted file mode 100644 index 340aa30..0000000 --- a/build/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Pyproject.nix build infrastructure - -Pyproject.nix can be used with nixpkgs `buildPythonPackage`/`packageOverrides`/`withPackages`, but also implements it's own build infrastructure. -It is experimental, and mainly targeted at expert users such as 2nix authors. If you're looking for packaging guidance you're in the wrong place. - -This fixes many problems existing in the Nixpkgs Python infrastructure such as: - - No dependency propagation - -No leaking of dependencies via PYTHONPATH - - No runtime deps at build time - -This causes much less rebuilds - - Support for circular dependencies - - No wrapping of Python bin's - -No strange venv breaking shell wrapper - - Proper virtualenvs only - -Meaning that applications that spawns children using sys.executable works fine. - - Hermetic bootstrap - -Nixpkgs breaks if you override any bootstrap dependencies with wheels. - - Manual cross splicing - -Meaning you can override build-time and runtime deps separately. diff --git a/build/bootstrap.nix b/build/bootstrap.nix index f3fef58..e94ee85 100644 --- a/build/bootstrap.nix +++ b/build/bootstrap.nix @@ -1,6 +1,5 @@ { stdenv, - lib, python, pyprojectInstallHook, pyprojectBytecodeHook, diff --git a/build/checks.nix b/build/checks.nix index 385a76b..0c6482d 100644 --- a/build/checks.nix +++ b/build/checks.nix @@ -46,7 +46,7 @@ in } ) { }; - # Bootstrap dependencies need to pyprojectBootstrapHook + # Bootstrap dependencies need to use pyprojectBootstrapHook overriden-bootstrap-dep = let overlay = final: _prev: { diff --git a/build/lib/renderers.nix b/build/lib/renderers.nix index 313b1fb..6fcf04d 100644 --- a/build/lib/renderers.nix +++ b/build/lib/renderers.nix @@ -16,10 +16,20 @@ let in { + /* + Renders a project as an argument that can be passed to stdenv.mkDerivation. + + Evaluates PEP-508 environment markers to select correct dependencies for the platform but does not validate version constraints. + + Type: mkDerivation :: AttrSet -> AttrSet + */ mkDerivation = { + # Loaded pyproject.nix project project, + # PEP-508 environment environ, + # Extras to enable (markers only, `optional-dependencies` are not enabled by default) extras ? [ ], }: let diff --git a/build/lib/resolvers.nix b/build/lib/resolvers.nix index 8161903..4ab8de5 100644 --- a/build/lib/resolvers.nix +++ b/build/lib/resolvers.nix @@ -2,7 +2,6 @@ let inherit (lib) - filterAttrs concatMap attrNames elemAt @@ -11,18 +10,18 @@ let genAttrs ; - # Most of the packages in pkgs/ are build-systems that we can use to create the memo - knownPackages = attrNames (filterAttrs (_: type: type == "directory") (builtins.readDir ../pkgs)); - in { /* Resolve dependencies using a non-circular supporting approach. - This implementation is faster than the one supporting circular dependencies - resolveNonCyclic is intended to resolve build-system dependencies. + This implementation is faster than the one supporting circular dependencies, and is memoized. + + `resolveNonCyclic` is intended to resolve build-system dependencies. */ resolveNonCyclic = + # List of package names to memoize + memoNames: # Package set to resolve packages from set: let @@ -44,7 +43,7 @@ in ) extras; # Memoise known build systems with no extras enabled for better performance - memo = genAttrs knownPackages (name: recurse' name [ ]); + memo = genAttrs memoNames (name: recurse' name [ ]); recurse = name: extras: if extras == [ ] then (memo.${name} or (recurse' name [ ])) else recurse' name extras; @@ -55,17 +54,16 @@ in /* Resolve dependencies using a cyclic supporting approach. - resolveCyclic is intended to resolve virtualenv dependencies. + `resolveCyclic` is intended to resolve virtualenv dependencies. */ resolveCyclic = - let - mkKey = key: { inherit key; }; - in # Package set to resolve packages from set: - # Attribute set of dependencies -> extras { requests = [ "socks" ]; } + # Attribute set of dependencies -> extras `{ requests = [ "socks" ]; }` spec: let + mkKey = key: { inherit key; }; + # Resolve spec recursively closure' = genericClosure { startSet = concatMap ( diff --git a/build/packages.nix b/build/packages.nix index 2237237..90c9834 100644 --- a/build/packages.nix +++ b/build/packages.nix @@ -3,10 +3,19 @@ let inherit (resolvers) resolveCyclic resolveNonCyclic; inherit (lib) makeScope; + mkPkgs' = import ./pkgs { inherit lib; }; + + # Build-system package names to memoise + memoNames = lib.attrNames (mkPkgs' { + # Hack: We only need attrNames + pyprojectBootstrapHook = null; + callPackage = null; + }); + mkResolveBuildSystem = set: let - resolveNonCyclic' = resolveNonCyclic set; + resolveNonCyclic' = resolveNonCyclic memoNames set; in spec: map (name: set.${name}) ( @@ -24,11 +33,7 @@ let mkResolveVirtualEnv = set: spec: map (name: set.${name}) (resolveCyclic set spec); - pkgsFun = - let - mkPkgs = import ./pkgs { inherit lib; }; - in - final: mkPkgs { inherit (final) callPackage pyprojectBootstrapHook; }; + pkgsFun = final: mkPkgs' { inherit (final) callPackage pyprojectBootstrapHook; }; in diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 624c806..009187b 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -8,13 +8,14 @@ - [requirements.txt](./use-cases/requirements.md) - [FAQ](./FAQ.md) -# Reference +# Library reference - [User facing APIs](./reference.md) - [project](./lib/project.nix) - - [renderers](./lib/renderers.nix) - - [validators](./lib/validators.nix) - [scripts](./lib/scripts.nix) + - [nixpkgs](./nixpkgs-build.md) + - [renderers](./lib/renderers.nix) + - [validators](./lib/validators.nix) - [Standards APIs](./reference.md) - [pep440](./lib/pep440.nix) - [pep508](./lib/pep508.nix) @@ -29,6 +30,11 @@ - [eggs](./lib/eggs.nix) - [pip](./lib/pip.nix) +- [Build](./build.md) + - [lib]() + - [renderers](./build/lib/renderers.nix) + - [resolvers](./build/lib/resolvers.nix) + # Contributing - [Hacking](./HACKING.md) diff --git a/doc/src/build b/doc/src/build new file mode 120000 index 0000000..8fdb6a2 --- /dev/null +++ b/doc/src/build @@ -0,0 +1 @@ +../../build \ No newline at end of file diff --git a/doc/src/build.md b/doc/src/build.md new file mode 100644 index 0000000..59467aa --- /dev/null +++ b/doc/src/build.md @@ -0,0 +1,108 @@ +# Build + +
+The pyproject.nix build infrastructure is brand new and experimental. +At this time it's mainly targeted at python2nix authors, and is being tested in uv2nix. +
+ +Pyproject.nix can be used with nixpkgs `buildPythonPackage`/`packageOverrides`/`withPackages`, but also implements it's own build infrastructure that fixes many structural problems with the nixpkgs implementation. + +## Problems with nixpkgs Python builders + +Nixpkgs Python infrastucture relies on [dependency propagation](https://nixos.org/manual/nixpkgs/unstable/#ssec-stdenv-dependencies-propagated). +The propagation mechanism works through making dependencies available to the builder at build-time, and recording their Nix store paths in `$out/nix-support/propagated-build-inputs`. +[Setup hooks](https://nixos.org/manual/nixpkgs/unstable/#ssec-setup-hooks) are then used to add these packages to `$PYTHONPATH` for discovery by the Python interpreter which adds everything from `$PYTHONPATH` to `sys.path` at startup. + +This causes several issues downstream. + +### `$PYTHONPATH` leaking into unrelated builds + +Consider the following development shell using nixpkgs Python builders: +``` nix +let + pkgs = import { }; + pythonEnv = pkgs.python3.withPackages(ps: [ ps.requests ]); +in pkgs.mkShell { + packages = [ + pkgs.remarshal + pythonEnv + ]; +} +``` + +Any Python package, such as `remarshal`, will have their dependencies leaking into `$PYTHONPATH`, making undeclared dependencies available to the Python interpreter. +Making matters even worse: Any dependency on `$PYTHONPATH` takes precedence over virtualenv installed dependencies! + +### Infinite recursions + +Nix dependency graphs are required to be a [DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph), but Python dependencies can be cyclic. +Dependency propagation is inherently incompatible with cyclic dependencies. +In nixpkgs this is commonly worked around by patching packages in various ways. + +## Solution presented by pyproject.nix's builders + +The solution is to decouple the runtime dependency graph from the build time one, by putting runtime dependencies in [passthru](https://nixos.org/manual/nixpkgs/unstable/#chap-passthru): +``` nix +stdenv.mkDerivation { + pname = "setuptools-scm"; + version = "8.1.0"; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/4f/a4/00a9ac1b555294710d4a68d2ce8dfdf39d72aa4d769a7395d05218d88a42/setuptools_scm-8.1.0.tar.gz"; + hash = ""; + }; + + passthru = { + dependencies = { + packaging = [ ]; + setuptools = [ ]; + }; + + optional-dependencies = { + toml = { toml = [ ]; }; + rich = { rich = [ ]; }; + }; + }; + + nativeBuildInputs = [ + pyprojectHook + ] ++ resolveBuildSystem ( + { + setuptools = [ ]; + } + ); +} +``` + +### Resolving + +Because runtime dependencies are not propagated every package needs to resolve the runtime dependencies of their build-system's. + +Additionally packages can't simply be consumed, but must be aggregated into a virtual environment to be useful: +``` nix +{ pyproject-nix, pkgs }: + +let + python = pkgs.python312; + + # Inject your own packages on top with overrideScope + pythonSet = pkgs.callPackage pyproject-nix.build.packages { + inherit python; + }; + +in pythonSet.pythonPackagesHostHost.mkVirtualEnv "test-venv" { + build = [ ]; +} +``` + +### Cyclic dependencies + +Cyclic dependencies are supported thanks to the resolver returning a flat list of required Python packages. +For performance reasons two solvers are implemented: + +- One that does not support cyclic dependencies + This is a much more performant resolver used by resolveBuildSystem and has all known build-systems memoized. + +- One that does support cyclic dependencies + Used to resolve virtual environments + +It's possible to override the resolver used entirely, so even though cyclic build-system's are not supported by default, it can be done with overrides. diff --git a/doc/src/nixpkgs-build.md b/doc/src/nixpkgs-build.md new file mode 100644 index 0000000..23b3347 --- /dev/null +++ b/doc/src/nixpkgs-build.md @@ -0,0 +1,3 @@ +# Build infrastructures + +Pyproject.nix can be used with nixpkgs `buildPythonPackage`/`packageOverrides`/`withPackages`, but also implements it's [own build infrastructure](./build.md) that fixes many structural problems with the nixpkgs implementation. From f968cbf76940691999ad5b67b550a02a1fa977ad Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Sep 2024 05:19:32 +0000 Subject: [PATCH 07/22] build: Package pytest-runner explicitly It was removed from nixpkgs --- build/pkgs/pytest-runner/default.nix | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/build/pkgs/pytest-runner/default.nix b/build/pkgs/pytest-runner/default.nix index 1251ad7..6c1b81e 100644 --- a/build/pkgs/pytest-runner/default.nix +++ b/build/pkgs/pytest-runner/default.nix @@ -1,16 +1,17 @@ { stdenv, - python3Packages, + fetchurl, pyprojectHook, resolveBuildSystem, }: stdenv.mkDerivation { - inherit (python3Packages.pytest-runner) - pname - version - src - meta - ; + pname = "pytest-runner"; + version = "6.0.1"; + + src = fetchurl { + url = "https://files.pythonhosted.org/packages/d7/7d/60976d532519c3a0b41e06a59ad60949e2be1af937cf02738fec91bfd808/pytest-runner-6.0.1.tar.gz"; + hash = "sha256-cNRzlYWnAI83v0kzwBP9sye4h4paafy7MxbIiILw9Js="; + }; nativeBuildInputs = [ From 55f2f323e4d68dadbe59255e53f3577007c0d7bc Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 23 Sep 2024 23:06:20 +0000 Subject: [PATCH 08/22] build: Use isBootstrapPackage to inject bootstrap hook when required --- build/default.nix | 2 +- build/packages.nix | 8 ++++++-- build/pkgs/default.nix | 37 ++++++++++++++++++------------------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/build/default.nix b/build/default.nix index e007ecc..2cdfba7 100644 --- a/build/default.nix +++ b/build/default.nix @@ -3,7 +3,7 @@ lib.fix (self: { packages = import ./packages.nix { inherit (self.lib) resolvers; - inherit lib; + inherit pyproject-nix lib; }; lib = import ./lib { inherit lib pyproject-nix; }; }) diff --git a/build/packages.nix b/build/packages.nix index 90c9834..c1da0de 100644 --- a/build/packages.nix +++ b/build/packages.nix @@ -1,9 +1,13 @@ -{ lib, resolvers }: +{ + lib, + pyproject-nix, + resolvers, +}: let inherit (resolvers) resolveCyclic resolveNonCyclic; inherit (lib) makeScope; - mkPkgs' = import ./pkgs { inherit lib; }; + mkPkgs' = import ./pkgs { inherit pyproject-nix lib; }; # Build-system package names to memoise memoNames = lib.attrNames (mkPkgs' { diff --git a/build/pkgs/default.nix b/build/pkgs/default.nix index 78e975c..62c76c3 100644 --- a/build/pkgs/default.nix +++ b/build/pkgs/default.nix @@ -1,26 +1,25 @@ -{ lib }: +{ lib, pyproject-nix }: let + inherit (pyproject-nix.build.lib) isBootstrapPackage; + inherit (lib) mapAttrs filterAttrs; + inherit (builtins) readDir; + # List all packages in directory - paths = lib.filterAttrs (_name: type: type == "directory") (builtins.readDir ./.); + paths = filterAttrs (_name: type: type == "directory") (readDir ./.); in { callPackage, pyprojectBootstrapHook }: # Automatically call all packages -(lib.mapAttrs (name: _: callPackage (./. + "/${name}") { }) paths) -// - # Override bootstrap packages with bootstrap hook - { - build = callPackage ./build { - pyprojectHook = pyprojectBootstrapHook; - }; - flit-core = callPackage ./flit-core { - pyprojectHook = pyprojectBootstrapHook; - }; - packaging = callPackage ./packaging { - pyprojectHook = pyprojectBootstrapHook; - }; - pyproject-hooks = callPackage ./pyproject-hooks { - pyprojectHook = pyprojectBootstrapHook; - }; - } +(mapAttrs ( + name: _: + callPackage (./. + "/${name}") ( + # Override bootstrap packages with bootstrap hook + if isBootstrapPackage name then + { + pyprojectHook = pyprojectBootstrapHook; + } + else + { } + ) +) paths) From c1c96f498f7274b6164e0c76470441b671fc5b91 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 00:08:48 +0000 Subject: [PATCH 09/22] build: Add check building all packages into a single venv --- build/checks.nix | 27 +++++++++++++- build/hooks/make-venv.py | 81 ++++++++++++++++++++++++++++------------ dev/treefmt.nix | 4 ++ lib/fixtures/pep723.py | 1 + 4 files changed, 89 insertions(+), 24 deletions(-) diff --git a/build/checks.nix b/build/checks.nix index 0c6482d..a52c9bb 100644 --- a/build/checks.nix +++ b/build/checks.nix @@ -1,7 +1,12 @@ -{ pyproject-nix, pkgs }: +{ + pyproject-nix, + lib, + pkgs, +}: let inherit (pyproject-nix.build.lib) isBootstrapPackage; + inherit (lib) filter attrValues isDerivation; python = pkgs.python312; @@ -80,4 +85,24 @@ in pythonSet'.pythonPackagesHostHost.mkVirtualEnv "overriden-bootstrap-venv" { build = [ ]; }; + + full-set = + let + pythonSetDrvs = filter isDerivation (attrValues pythonSet.pythonPackagesHostHost); + hooks = attrValues pythonSet.pythonPackagesHostHost.hooks; + pythonDrvs = filter ( + drv: + !lib.elem drv hooks + && !lib.elem (drv.pname or drv.name) [ + "pyproject-hook" + "python3" + "stdenv-linux" + ] + ) pythonSetDrvs; + + full-set-venv = pythonSet.pythonPackagesHostHost.mkVirtualEnv "test-venv" ( + lib.listToAttrs (map (drv: lib.nameValuePair (drv.pname or drv.name) [ ]) pythonDrvs) + ); + in + full-set-venv; } diff --git a/build/hooks/make-venv.py b/build/hooks/make-venv.py index 37141ce..ea205d7 100644 --- a/build/hooks/make-venv.py +++ b/build/hooks/make-venv.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 -from venv import EnvBuilder -from pathlib import Path +import os import os.path import shutil import stat import sys -import os - +from pathlib import Path +from venv import EnvBuilder EXECUTABLE = os.path.basename(sys.executable) PYTHON_VERSION = ".".join((str(sys.version_info.major), str(sys.version_info.minor))) @@ -43,7 +42,7 @@ def write_bin_dir(bin_dir: Path, bin_out: Path) -> None: with open(bin_file_out, "wb") as f_out: f_out.write(out_shebang) shutil.copyfileobj(f_in, f_out) - os.chmod(bin_file_out, st_mode) # Copy mode + os.chmod(bin_file_out, st_mode) # Copy mode continue # Symlink anything else @@ -51,16 +50,50 @@ def write_bin_dir(bin_dir: Path, bin_out: Path) -> None: # Special rules for site-packages: -# - Rewrite package-local symlinks to venv -# - Directly symlink regardless of type -def write_site_packages(site_packages: Path, out_site_packages: Path) -> None: - for site_pkg_file in os.listdir(site_packages): - src_file = site_packages.joinpath(site_pkg_file) - dst_file = out_site_packages.joinpath(site_pkg_file) - if stat.S_ISLNK(src_file.lstat().st_mode): - shutil.copy(src_file, dst_file, follow_symlinks=False) - else: - os.symlink(src_file, dst_file) +# - Symlink packages +# - If a collision is found replace symlink with a directory containing all members of old directory, merged with new directory +def write_site_packages(src: Path, dst: Path) -> None: + src_stat = src.lstat() + + # Copy symbolic links + if stat.S_ISLNK(src_stat.st_mode): + shutil.copy(src, dst, follow_symlinks=False) + + # Recursively link directories + elif stat.S_ISDIR(src_stat.st_mode): + try: + dst_st_mode = dst.lstat().st_mode + except FileNotFoundError: + # No collision (yet?), simply place symlink + os.symlink(src, dst) + return + + # If we have an existing symlink "upgrade" it into a directory + if stat.S_ISLNK(dst_st_mode): + dst_target = dst.readlink() + if not stat.S_ISDIR(dst_target.stat().st_mode): + raise ValueError("Source of site-packages collision was not a directory. Unable to merge.") + + # "Upgrade" symlink to directory + dst.unlink() + dst.mkdir() + + # Symlink files from old target into directory + for src_file_nested in dst_target.iterdir(): + dst.joinpath(src_file_nested.name).symlink_to(src_file_nested) + + try: + dst.mkdir() + except FileExistsError: + pass + + # Symlink files from new target + for src_file_nested in src.iterdir(): + write_site_packages(src_file_nested, dst.joinpath(src_file_nested.name)) + + # For any other type, symlink + else: + os.symlink(src, dst) def link_dependency(dep_root: Path, out_root: Path) -> None: @@ -92,7 +125,7 @@ def _link(root: Path, out: Path) -> None: st_mode = path.lstat().st_mode if stat.S_ISLNK(st_mode): - shutil.copy(path, out.joinpath(path.name)) + shutil.copy(path, out.joinpath(path.name), follow_symlinks=False) elif stat.S_ISREG(st_mode): os.symlink(path, out.joinpath(path.name)) elif stat.S_ISDIR(st_mode): @@ -128,9 +161,11 @@ def main(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument("out", help="Virtualenv output directory") - arg_parser.add_argument("--python", help="Python to link virtualenv to", default=os.path.dirname(os.path.dirname(sys.executable))) - arg_parser.add_argument("--env", action="append", help="Source dependencies from environment variable") - arg_parser.add_argument("--deps", action="append", help="Source dependencies from colon separated list") + arg_parser.add_argument( + "--python", help="Python to link virtualenv to", default=os.path.dirname(os.path.dirname(sys.executable)) + ) + arg_parser.add_argument("--env", action="append", help="Source dependencies from environment variable") + arg_parser.add_argument("--deps", action="append", help="Source dependencies from colon separated list") args = arg_parser.parse_args() @@ -138,11 +173,11 @@ def main(): python_root = Path(args.python) python_executable = python_root.joinpath("bin", EXECUTABLE) - dependencies: list[Path] = [] # List of dependency roots - seen_roots: set[str] = set() # Keep track of unique dependency roots + dependencies: list[Path] = [] # List of dependency roots + seen_roots: set[str] = set() # Keep track of unique dependency roots # Populate dependencies from precisely passed options - for dep_roots in (args.deps or []): + for dep_roots in args.deps or []: for dep_root in dep_roots.split(":"): if dep_root in seen_roots: continue @@ -150,7 +185,7 @@ def main(): dependencies.append(Path(dep_root)) # Populate dependencies from env - for env_var in (args.env or []): + for env_var in args.env or []: try: env_value = os.environ[env_var] except KeyError: diff --git a/dev/treefmt.nix b/dev/treefmt.nix index d24403f..d95ecf5 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -6,4 +6,8 @@ _: { programs.deadnix.enable = true; programs.statix.enable = true; programs.nixfmt.enable = true; + + # Python + programs.ruff-format.enable = true; + programs.ruff-check.enable = true; } diff --git a/lib/fixtures/pep723.py b/lib/fixtures/pep723.py index 9723000..60b5c8c 100644 --- a/lib/fixtures/pep723.py +++ b/lib/fixtures/pep723.py @@ -1,3 +1,4 @@ +# ruff: noqa # /// script # requires-python = ">=3.11" # dependencies = [ From 203ddccbbbed69a971f776070e99f4765a9aa8b9 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 00:17:24 +0000 Subject: [PATCH 10/22] treefmt: Enable mypy --- dev/treefmt.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/treefmt.nix b/dev/treefmt.nix index d95ecf5..e70977e 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -10,4 +10,5 @@ _: { # Python programs.ruff-format.enable = true; programs.ruff-check.enable = true; + programs.mypy.enable = true; } From 027494d2bcfb9d3e5abc365a7660eb56c8b1c353 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 00:24:06 +0000 Subject: [PATCH 11/22] doc: Add build lib root --- build/lib/default.nix | 14 +++++++++----- doc/src/SUMMARY.md | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/build/lib/default.nix b/build/lib/default.nix index 447867e..96861e2 100644 --- a/build/lib/default.nix +++ b/build/lib/default.nix @@ -6,12 +6,12 @@ in fix ( self: - mapAttrs (_: path: import path ({ inherit pyproject-nix lib; } // self)) { - renderers = ./renderers.nix; - resolvers = ./resolvers.nix; - } - // { + { + /* + Check if a package is a bootstrap package by it's name. + This needs to be used by lockfile consumers to check if a package needs pyprojectBootstrapHook instead of pyprojectHook. + */ isBootstrapPackage = flip elem [ "flit-core" "pyproject-hooks" @@ -19,4 +19,8 @@ fix ( "build" ]; } + // mapAttrs (_: path: import path ({ inherit pyproject-nix lib; } // self)) { + renderers = ./renderers.nix; + resolvers = ./resolvers.nix; + } ) diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 009187b..29b5ead 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -31,7 +31,7 @@ - [pip](./lib/pip.nix) - [Build](./build.md) - - [lib]() + - [lib](./build/lib/default.nix) - [renderers](./build/lib/renderers.nix) - [resolvers](./build/lib/resolvers.nix) From 7c608e4998a57b4d6539603958eef4e952572db2 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 00:28:58 +0000 Subject: [PATCH 12/22] doc: Update intro --- doc/src/introduction.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/src/introduction.md b/doc/src/introduction.md index 583541b..a86c4b1 100644 --- a/doc/src/introduction.md +++ b/doc/src/introduction.md @@ -5,13 +5,15 @@ Pyproject.nix is a collection of Nix utilities to work with Python project metadata in [Nix](https://nixos.org/). It mainly targets [PEP-621](https://peps.python.org/pep-0621/) compliant `pyproject.toml` files and data formats, but also implement support for other & legacy formats such as [Poetry](https://python-poetry.org/) & `requirements.txt`. -Pyproject.nix aims to be a swiss army knife of simple customizable utilities that works together with the [nixpkgs Python infrastructure](https://nixos.org/manual/nixpkgs/stable/#python). +Pyproject.nix aims to be a swiss army knife of simple customizable utilities that works either together with the [nixpkgs Python infrastructure](https://nixos.org/manual/nixpkgs/stable/#python), or [our own build infrastructure](./build.md). ## Foreword This documentation only helps you to get started with `pyproject.nix`. As it's a toolkit with many use cases not every use case can be documented fully. +First and foremost `pyproject.nix` is a _Python metadata toolkit_, and not a 2nix tool. + This documentation is centered around packaging Python applications & managing development environments. For other use cases see the reference documentation. @@ -42,7 +44,7 @@ This can be useful to check that a package set is compilant with the specificati ### [Renderers](./lib/renderers.md) -A `renderer` takes a `project` together with a Python interpreter derivation and renders it into a form understood by various pieces of nixpkgs Python infrastructure. +A `renderer` takes a `project` together with a Python interpreter derivation and renders it into a form understood by various pieces of Python infrastructure. For example: The `buildPythonPackage` renderer returns an attribute set that can be passed to either nixpkgs function `buildPythonPackage` or `buildPythonApplication`. From 766c8603749743451556aec079ed86a369890dbe Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 03:38:05 +0000 Subject: [PATCH 13/22] build: Add editable rendering --- build/{checks.nix => checks/default.nix} | 102 +++++++++++++- build/checks/fixtures/myapp/.python-version | 1 + build/checks/fixtures/myapp/README.md | 3 + build/checks/fixtures/myapp/pyproject.toml | 19 +++ .../fixtures/myapp/src/myapp/__init__.py | 2 + build/lib/renderers.nix | 127 +++++++++++++++++- build/pkgs/tomli-w/default.nix | 22 +++ flake.nix | 2 +- 8 files changed, 275 insertions(+), 3 deletions(-) rename build/{checks.nix => checks/default.nix} (52%) create mode 100644 build/checks/fixtures/myapp/.python-version create mode 100644 build/checks/fixtures/myapp/README.md create mode 100644 build/checks/fixtures/myapp/pyproject.toml create mode 100644 build/checks/fixtures/myapp/src/myapp/__init__.py create mode 100644 build/pkgs/tomli-w/default.nix diff --git a/build/checks.nix b/build/checks/default.nix similarity index 52% rename from build/checks.nix rename to build/checks/default.nix index a52c9bb..467aab9 100644 --- a/build/checks.nix +++ b/build/checks/default.nix @@ -5,7 +5,7 @@ }: let - inherit (pyproject-nix.build.lib) isBootstrapPackage; + inherit (pyproject-nix.build.lib) isBootstrapPackage renderers; inherit (lib) filter attrValues isDerivation; python = pkgs.python312; @@ -19,6 +19,13 @@ let build = [ ]; }; + # Test fixture + myapp = pyproject-nix.lib.project.loadPyproject { + projectRoot = ./fixtures/myapp; + }; + + testEnviron = pyproject-nix.lib.pep508.mkEnviron python; + in { @@ -105,4 +112,97 @@ in ); in full-set-venv; + + mkderivation = + let + testSet = pythonSet.pythonPackagesHostHost.overrideScope ( + final: _prev: { + myapp = final.callPackage ( + { + stdenv, + pyprojectHook, + resolveBuildSystem, + }: + stdenv.mkDerivation ( + renderers.mkDerivation + { + project = myapp; + environ = testEnviron; + } + { + inherit pyprojectHook resolveBuildSystem; + } + ) + ) { }; + } + ); + + venv = testSet.mkVirtualEnv "render-mkderivation-env" { + myapp = [ "toml" ]; + }; + in + pkgs.runCommand "render-mkderivation-test" { nativeBuildInputs = [ venv ]; } '' + # Assert that extra was enabled + python -c "import tomli_w" + + # Script from myapp + hello + + touch $out + ''; + + mkderivation-editable = + let + testSet = pythonSet.pythonPackagesHostHost.overrideScope ( + final: _prev: { + myapp = final.callPackage ( + { + python, + stdenv, + pyprojectHook, + resolveBuildSystem, + pythonPackagesBuildHost, + }: + stdenv.mkDerivation ( + renderers.mkDerivationEditable + { + project = myapp; + environ = testEnviron; + root = "$NIX_BUILD_TOP/src"; + } + { + inherit + python + pyprojectHook + resolveBuildSystem + pythonPackagesBuildHost + ; + } + ) + ) { }; + } + ); + + venv = testSet.mkVirtualEnv "render-mkderivation-editable-env" { + myapp = [ ]; + }; + + in + pkgs.runCommand "render-mkeditable" { nativeBuildInputs = [ venv ]; } '' + # Unpack sources into build + cp -r ${./fixtures/myapp}/* . + chmod +w -R src + + hello | grep "Hello from myapp" + + cat > src/myapp/__init__.py < None: + print("Hello from editable!") + EOF + + hello | grep "Hello from editable" + + touch $out + ''; + } diff --git a/build/checks/fixtures/myapp/.python-version b/build/checks/fixtures/myapp/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/build/checks/fixtures/myapp/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/build/checks/fixtures/myapp/README.md b/build/checks/fixtures/myapp/README.md new file mode 100644 index 0000000..4d339d9 --- /dev/null +++ b/build/checks/fixtures/myapp/README.md @@ -0,0 +1,3 @@ +# myapp + +A simple application used to integration test build renderers diff --git a/build/checks/fixtures/myapp/pyproject.toml b/build/checks/fixtures/myapp/pyproject.toml new file mode 100644 index 0000000..3ce102d --- /dev/null +++ b/build/checks/fixtures/myapp/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "myapp" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "zipp" +] + +[project.optional-dependencies] +toml = ["tomli-w"] + +[project.scripts] +hello = "myapp:hello" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/build/checks/fixtures/myapp/src/myapp/__init__.py b/build/checks/fixtures/myapp/src/myapp/__init__.py new file mode 100644 index 0000000..93c29aa --- /dev/null +++ b/build/checks/fixtures/myapp/src/myapp/__init__.py @@ -0,0 +1,2 @@ +def hello() -> None: + print("Hello from myapp!") diff --git a/build/lib/renderers.nix b/build/lib/renderers.nix index 6fcf04d..138747a 100644 --- a/build/lib/renderers.nix +++ b/build/lib/renderers.nix @@ -7,12 +7,18 @@ let concatMap groupBy ; + inherit (pyproject-nix.lib.renderers) meta; inherit (pyproject-nix.lib) pep621; # Make a dependency specification attrset from a list of dependencies mkSpec = dependencies: mapAttrs (_: concatMap (dep: dep.extras)) (groupBy (dep: dep.name) dependencies); + fallbackBuildSystems = map pyproject-nix.lib.pep508.parseString [ + "setuptools" + "wheel" + ]; + in { @@ -49,7 +55,7 @@ in pyprojectHook ] ++ resolveBuildSystem (mkSpec filteredDeps.build-systems); - meta = pyproject-nix.lib.renderers.meta { + meta = meta { inherit project; }; } @@ -59,4 +65,123 @@ in inherit (pyproject.project) name; } // optionalAttrs (project.projectRoot != null) { src = project.projectRoot; }; + + /* + Renders a project as an argument that can be passed to stdenv.mkDerivation. + + Evaluates PEP-508 environment markers to select correct dependencies for the platform but does not validate version constraints. + + Type: mkDerivation :: AttrSet -> AttrSet + */ + mkDerivationEditable = + { + # Loaded pyproject.nix project + project, + # PEP-508 environment + environ, + # Extras to enable (markers only, `optional-dependencies` are not enabled by default) + extras ? [ ], + # Editable root directory as a string + root ? toString ( + # Prefer src layout if available + if lib.pathExists (project.projectRoot + "/src") then + (project.projectRoot + "/src") + # Otherwise assume project root is editable root + else + project.projectRoot + ), + }: + let + filteredDeps = pep621.filterDependenciesByEnviron environ extras project.dependencies; + in + { + python, + pyprojectHook, + pythonPackagesBuildHost, + resolveBuildSystem, + }: + let + project' = project.pyproject.project; + pname = project'.name; + + # Synthetic pyproject.toml + # + # We don't use the provided build-system to build an editable package, we use hatchling. + pyproject = { + # PEP-621 project table + project = + { + # Both name and version are required. + inherit (project') name version; + } + // optionalAttrs (project' ? dependencies) { + inherit (project') dependencies; + } + // optionalAttrs (project' ? optional-dependencies) { + inherit (project') optional-dependencies; + } + // optionalAttrs (project' ? scripts) { + inherit (project') scripts; + } + // optionalAttrs (project' ? gui-scripts) { + inherit (project') gui-scripts; + } + // optionalAttrs (project' ? entry-points) { + inherit (project') entry-points; + }; + + # Allow empty package + tool.hatch.build.targets.wheel.bypass-selection = true; + + # Include our editable pointer file in build + tool.hatch.build.targets.wheel.force-include."_${pname}.pth" = "_${pname}.pth"; + + # Build editable package using hatchling + build-system = { + requires = [ "hatchling" ]; + build-backend = "hatchling.build"; + }; + }; + in + { + inherit pname; + + passthru = { + dependencies = mkSpec ( + filteredDeps.dependencies + ++ ( + if (filteredDeps.build-systems != [ ]) then filteredDeps.build-systems else fallbackBuildSystems + ) + ); + optional-dependencies = mapAttrs (_: mkSpec) filteredDeps.extras; + }; + + # Convert created JSON format pyproject.toml into TOML and include a generated pth file + unpackPhase = '' + env PYTHONPATH=${pythonPackagesBuildHost.tomli-w}/${python.sitePackages} python -c "import json, tomli_w; print(tomli_w.dumps(json.load(open('$pyprojectContentsPath'))))" > pyproject.toml + echo 'import os.path, sys; sys.path.insert(0, os.path.expandvars("${root}"))' > _${pname}.pth + ''; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + hatchling = [ ]; + }; + + pyprojectContents = builtins.toJSON pyproject; + passAsFile = [ "pyprojectContents" ]; + + meta = meta { + inherit project; + }; + } + // optionalAttrs (project' ? version) { + inherit (project') version; + } + // optionalAttrs (!project' ? version) { + name = pname; + }; + } diff --git a/build/pkgs/tomli-w/default.nix b/build/pkgs/tomli-w/default.nix new file mode 100644 index 0000000..070ab36 --- /dev/null +++ b/build/pkgs/tomli-w/default.nix @@ -0,0 +1,22 @@ +{ + stdenv, + python3Packages, + pyprojectHook, + resolveBuildSystem, +}: +stdenv.mkDerivation { + inherit (python3Packages.tomli-w) + pname + version + src + meta + ; + + nativeBuildInputs = + [ + pyprojectHook + ] + ++ resolveBuildSystem { + flit-core = [ ]; + }; +} diff --git a/flake.nix b/flake.nix index 9baf228..49a0fac 100644 --- a/flake.nix +++ b/flake.nix @@ -118,7 +118,7 @@ pkgs.callPackages ./test { pyproject = import ./default.nix { inherit lib; }; } )) // (lib.mapAttrs' (name: drv: lib.nameValuePair "build-${name}" drv) ( - pkgs.callPackages ./build/checks.nix { pyproject-nix = self; } + pkgs.callPackages ./build/checks { pyproject-nix = self; } )) // { formatter = pkgs.writeShellScript "fmt-check" '' From c095e1c427766aa0e35343622be10faf72fd81d5 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 21:46:20 +0000 Subject: [PATCH 14/22] build: pythonPackages(Build|Host)Host -> pythonPkgs(Build|Host)Host This is shorter and less likely to be mixed up with python(2|3|)Packages from nixpkgs. --- build/checks/default.nix | 22 +++++++++++----------- build/hooks/default.nix | 8 ++++---- build/hooks/make-venv.py | 34 ++++++++++++++++++++++++++++++++-- build/lib/renderers.nix | 4 ++-- build/lib/test_renderers.nix | 2 +- build/packages.nix | 36 ++++++++++++++++++------------------ doc/src/build.md | 2 +- 7 files changed, 69 insertions(+), 39 deletions(-) diff --git a/build/checks/default.nix b/build/checks/default.nix index 467aab9..f13e6c1 100644 --- a/build/checks/default.nix +++ b/build/checks/default.nix @@ -15,7 +15,7 @@ let inherit python; }; - testVenv = pythonSet.pythonPackagesHostHost.mkVirtualEnv "test-venv" { + testVenv = pythonSet.pythonPkgsHostHost.mkVirtualEnv "test-venv" { build = [ ]; }; @@ -39,7 +39,7 @@ in touch $out ''; - prebuilt-wheel = pythonSet.pythonPackagesHostHost.callPackage ( + prebuilt-wheel = pythonSet.pythonPkgsHostHost.callPackage ( { stdenv, fetchurl, @@ -84,19 +84,19 @@ in pythonSet' = pythonSet.overrideScope ( _final: prev: { - pythonPackagesBuildHost = prev.pythonPackagesBuildHost.overrideScope overlay; + pythonPkgsBuildHost = prev.pythonPkgsBuildHost.overrideScope overlay; } ); in - pythonSet'.pythonPackagesHostHost.mkVirtualEnv "overriden-bootstrap-venv" { + pythonSet'.pythonPkgsHostHost.mkVirtualEnv "overriden-bootstrap-venv" { build = [ ]; }; full-set = let - pythonSetDrvs = filter isDerivation (attrValues pythonSet.pythonPackagesHostHost); - hooks = attrValues pythonSet.pythonPackagesHostHost.hooks; + pythonSetDrvs = filter isDerivation (attrValues pythonSet.pythonPkgsHostHost); + hooks = attrValues pythonSet.pythonPkgsHostHost.hooks; pythonDrvs = filter ( drv: !lib.elem drv hooks @@ -107,7 +107,7 @@ in ] ) pythonSetDrvs; - full-set-venv = pythonSet.pythonPackagesHostHost.mkVirtualEnv "test-venv" ( + full-set-venv = pythonSet.pythonPkgsHostHost.mkVirtualEnv "test-venv" ( lib.listToAttrs (map (drv: lib.nameValuePair (drv.pname or drv.name) [ ]) pythonDrvs) ); in @@ -115,7 +115,7 @@ in mkderivation = let - testSet = pythonSet.pythonPackagesHostHost.overrideScope ( + testSet = pythonSet.pythonPkgsHostHost.overrideScope ( final: _prev: { myapp = final.callPackage ( { @@ -153,7 +153,7 @@ in mkderivation-editable = let - testSet = pythonSet.pythonPackagesHostHost.overrideScope ( + testSet = pythonSet.pythonPkgsHostHost.overrideScope ( final: _prev: { myapp = final.callPackage ( { @@ -161,7 +161,7 @@ in stdenv, pyprojectHook, resolveBuildSystem, - pythonPackagesBuildHost, + pythonPkgsBuildHost, }: stdenv.mkDerivation ( renderers.mkDerivationEditable @@ -175,7 +175,7 @@ in python pyprojectHook resolveBuildSystem - pythonPackagesBuildHost + pythonPkgsBuildHost ; } ) diff --git a/build/hooks/default.nix b/build/hooks/default.nix index 235d0ab..b6e6949 100644 --- a/build/hooks/default.nix +++ b/build/hooks/default.nix @@ -7,7 +7,7 @@ resolveBuildSystem, stdenv, hooks, - pythonPackagesBuildHost, + pythonPkgsBuildHost, }: let inherit (python) pythonOnBuildForHost isPy3k; @@ -42,7 +42,7 @@ in makeSetupHook { name = "pyproject-build-hook"; substitutions = { - inherit (pythonPackagesBuildHost) build; + inherit (pythonPkgsBuildHost) build; inherit pythonInterpreter; }; propagatedBuildInputs = resolveBuildSystem { @@ -74,11 +74,11 @@ in }; pyprojectPypaInstallHook = callPackage ( - { pythonPackagesBuildHost }: + { pythonPkgsBuildHost }: makeSetupHook { name = "pyproject-pypa-install-hook"; substitutions = { - inherit (pythonPackagesBuildHost) installer; + inherit (pythonPkgsBuildHost) installer; inherit pythonInterpreter pythonSitePackages; }; } ./pyproject-pypa-install-hook.sh diff --git a/build/hooks/make-venv.py b/build/hooks/make-venv.py index ea205d7..deda5fc 100644 --- a/build/hooks/make-venv.py +++ b/build/hooks/make-venv.py @@ -4,6 +4,7 @@ import shutil import stat import sys +import typing from pathlib import Path from venv import EnvBuilder @@ -16,6 +17,16 @@ dep_shebang = ("#!" + os.path.dirname(sys.executable)).encode() +# Compare the contents of two files by their file descriptors +def compare_fds(fa: typing.IO, fb: typing.IO) -> bool: + while True: + ba = fa.read(8192) + if ba != fb.read(8192): + return False + if not ba: + return True + + # Special rules for bin directory: # - Copy symlinks # - Rewrite scripts for shebangs @@ -30,15 +41,29 @@ def write_bin_dir(bin_dir: Path, bin_out: Path) -> None: # Copy symlinks if stat.S_ISLNK(st_mode): - shutil.copy(bin_file, bin_out.joinpath(bin_file.name), follow_symlinks=False) + try: + shutil.copy(bin_file, bin_file_out, follow_symlinks=False) + except FileExistsError: + # If the file exists but is pointing to the same symlink continue + if bin_file.resolve() != bin_file_out.resolve(): + raise # Rewrite script shebangs elif stat.S_ISREG(st_mode): # Check if file starts with Python shebang with open(bin_file, "rb") as f_in: shebang = f_in.read(len(dep_shebang)) + # If it does, rewrite it to venv interpreter if shebang == dep_shebang: + # Check if the destination file already exists + # If it does compare the contents of the files, modulo shebang + if bin_file_out.exists(): + with open(bin_file_out, "wb") as f_exist: + f_exist.read(len(dep_shebang)) + if not compare_fds(): + raise FileExistsError(f"File '{bin_file_out}' ") + with open(bin_file_out, "wb") as f_out: f_out.write(out_shebang) shutil.copyfileobj(f_in, f_out) @@ -46,7 +71,12 @@ def write_bin_dir(bin_dir: Path, bin_out: Path) -> None: continue # Symlink anything else - os.symlink(bin_file, bin_file_out) + try: + os.symlink(bin_file, bin_file_out) + except FileExistsError: + # If the file exists but is pointing to the same symlink continue + if bin_file.resolve() != bin_file_out.resolve(): + raise # Special rules for site-packages: diff --git a/build/lib/renderers.nix b/build/lib/renderers.nix index 138747a..f2fded0 100644 --- a/build/lib/renderers.nix +++ b/build/lib/renderers.nix @@ -97,7 +97,7 @@ in { python, pyprojectHook, - pythonPackagesBuildHost, + pythonPkgsBuildHost, resolveBuildSystem, }: let @@ -158,7 +158,7 @@ in # Convert created JSON format pyproject.toml into TOML and include a generated pth file unpackPhase = '' - env PYTHONPATH=${pythonPackagesBuildHost.tomli-w}/${python.sitePackages} python -c "import json, tomli_w; print(tomli_w.dumps(json.load(open('$pyprojectContentsPath'))))" > pyproject.toml + env PYTHONPATH=${pythonPkgsBuildHost.tomli-w}/${python.sitePackages} python -c "import json, tomli_w; print(tomli_w.dumps(json.load(open('$pyprojectContentsPath'))))" > pyproject.toml echo 'import os.path, sys; sys.path.insert(0, os.path.expandvars("${root}"))' > _${pname}.pth ''; diff --git a/build/lib/test_renderers.nix b/build/lib/test_renderers.nix index c2720ee..6788b3b 100644 --- a/build/lib/test_renderers.nix +++ b/build/lib/test_renderers.nix @@ -33,7 +33,7 @@ in }) { pyprojectHook = null; - inherit (pythonSet.pythonPackagesHostHost) resolveBuildSystem; + inherit (pythonSet.pythonPkgsHostHost) resolveBuildSystem; }; in rendered diff --git a/build/packages.nix b/build/packages.nix index c1da0de..c240f93 100644 --- a/build/packages.nix +++ b/build/packages.nix @@ -55,14 +55,14 @@ makeScope newScope ( newScope, python, stdenv, - pythonPackagesBuildHost, + pythonPkgsBuildHost, bootstrapHooks, - pythonPackagesFun, + pythonPkgsFun, }: makeScope newScope ( pkgsFinal: { - inherit python stdenv pythonPackagesBuildHost; + inherit python stdenv pythonPkgsBuildHost; # Pyproject hook used for bootstrap packages pyprojectBootstrapHook = pkgsFinal.pyprojectHook.override { @@ -70,7 +70,7 @@ makeScope newScope ( }; # Initialize dependency resolvers - resolveBuildSystem = mkResolveBuildSystem pythonPackagesBuildHost; + resolveBuildSystem = mkResolveBuildSystem pythonPkgsBuildHost; resolveVirtualEnv = mkResolveVirtualEnv pkgsFinal; # Make a virtual env from resolved dependencies @@ -102,12 +102,12 @@ makeScope newScope ( pyprojectWheelHook ; } - // pythonPackagesFun pkgsFinal + // pythonPkgsFun pkgsFinal ); bootstrapHooks = final.callPackage ./hooks { python = final.python.pythonOnBuildForHost; - resolveBuildSystem = mkResolveBuildSystem final.pythonPackagesBootstrap; + resolveBuildSystem = mkResolveBuildSystem final.pythonPkgsBootstrap; hooks = bootstrapHooks; }; @@ -116,12 +116,12 @@ makeScope newScope ( # Allows overriding Python by calling overrideScope on the outer scope inherit python; - pythonPackagesBootstrap = mkPythonSet { + pythonPkgsBootstrap = mkPythonSet { inherit (buildPackages) stdenv newScope; inherit bootstrapHooks; python = python.pythonOnBuildForHost; - pythonPackagesBuildHost = final.pythonPackagesBootstrap; - pythonPackagesFun = + pythonPkgsBuildHost = final.pythonPkgsBootstrap; + pythonPkgsFun = _: final.callPackage ./bootstrap.nix { inherit (bootstrapHooks) pyprojectInstallHook pyprojectBytecodeHook pyprojectOutputSetupHook; @@ -130,25 +130,25 @@ makeScope newScope ( }; # Python packages for the build host - pythonPackagesBuildHost = mkPythonSet { + pythonPkgsBuildHost = mkPythonSet { inherit (buildPackages) stdenv newScope; python = python.pythonOnBuildForHost; - inherit (final) pythonPackagesBuildHost; - bootstrapHooks = final.pythonPackagesBootstrap.hooks; - pythonPackagesFun = pkgsFun; + inherit (final) pythonPkgsBuildHost; + bootstrapHooks = final.pythonPkgsBootstrap.hooks; + pythonPkgsFun = pkgsFun; }; # Python packages for the target host - pythonPackagesHostHost = + pythonPkgsHostHost = # If we're not doing cross reference build host packages if stdenv.buildPlatform != stdenv.hostPlatform then mkPythonSet { - inherit (final) newScope pythonPackagesBuildHost; + inherit (final) newScope pythonPkgsBuildHost; inherit python stdenv; - bootstrapHooks = final.pythonPackagesBuildHost.hooks; - pythonPackagesFun = pkgsFun; + bootstrapHooks = final.pythonPkgsBuildHost.hooks; + pythonPkgsFun = pkgsFun; } else - final.pythonPackagesBuildHost; + final.pythonPkgsBuildHost; } ) diff --git a/doc/src/build.md b/doc/src/build.md index 59467aa..5e28159 100644 --- a/doc/src/build.md +++ b/doc/src/build.md @@ -89,7 +89,7 @@ let inherit python; }; -in pythonSet.pythonPackagesHostHost.mkVirtualEnv "test-venv" { +in pythonSet.pythonPkgsHostHost.mkVirtualEnv "test-venv" { build = [ ]; } ``` From 82eec0cbeebe0c51203e4d7911daba7702af1a5b Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 22:11:29 +0000 Subject: [PATCH 15/22] doc: Add notes on nixpkgs binary wrapping --- doc/src/build.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/src/build.md b/doc/src/build.md index 5e28159..6a02d54 100644 --- a/doc/src/build.md +++ b/doc/src/build.md @@ -39,6 +39,13 @@ Nix dependency graphs are required to be a [DAG](https://en.wikipedia.org/wiki/D Dependency propagation is inherently incompatible with cyclic dependencies. In nixpkgs this is commonly worked around by patching packages in various ways. +### Binary wrapping + +Nixpkgs Python builders uses wrappers for Python executables in `bin/`, these set environment variables `NIX_PYTHONPATH` & friends +This is picked up by the interpreter using a [sitecustomize.py](https://docs.python.org/3/library/site.html#module-sitecustomize) in the system `site-packages` directory. + +This means that any Python programs executing another child Python interpreter using `sys.executable` will have it's modules lost on import, as `sys.executable` isn't pointing to the environment created by `withPackages`. + ## Solution presented by pyproject.nix's builders The solution is to decouple the runtime dependency graph from the build time one, by putting runtime dependencies in [passthru](https://nixos.org/manual/nixpkgs/unstable/#chap-passthru): @@ -106,3 +113,7 @@ For performance reasons two solvers are implemented: Used to resolve virtual environments It's possible to override the resolver used entirely, so even though cyclic build-system's are not supported by default, it can be done with overrides. + +### Use virtualenvs + +Instead of binary wrappers & environment variables `pyproject.nix`'s builders use standard Python virtual environments. From a25e69c5380ffdca3fdf5a37d57a60e9ac640e4e Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 24 Sep 2024 22:40:39 +0000 Subject: [PATCH 16/22] build: Add error message when editable root is not a string, or a string pointing to a store path --- build/lib/renderers.nix | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/build/lib/renderers.nix b/build/lib/renderers.nix index f2fded0..2757d5d 100644 --- a/build/lib/renderers.nix +++ b/build/lib/renderers.nix @@ -6,9 +6,16 @@ let mapAttrs concatMap groupBy + pathExists + isString + assertMsg + hasPrefix + optionalString + inPureEvalMode ; inherit (pyproject-nix.lib.renderers) meta; inherit (pyproject-nix.lib) pep621; + inherit (builtins) storeDir; # Make a dependency specification attrset from a list of dependencies mkSpec = @@ -84,13 +91,24 @@ in # Editable root directory as a string root ? toString ( # Prefer src layout if available - if lib.pathExists (project.projectRoot + "/src") then + if pathExists (project.projectRoot + "/src") then (project.projectRoot + "/src") # Otherwise assume project root is editable root else project.projectRoot ), }: + assert isString root; + assert assertMsg (!hasPrefix storeDir root) '' + Editable root was passed as a Nix store path string. + + ${optionalString (inPureEvalMode) '' + This is most likely because you are using Flakes, and are automatically inferring the editable root from projectRoot. + Flakes are copied to the Nix store on evaluation. This can temporarily be worked around using --impure. + ''} + + Pass editable root either as a string pointing to an absolute path non-store path, or use environment variables for relative paths. + ''; let filteredDeps = pep621.filterDependenciesByEnviron environ extras project.dependencies; in From 3e1df258e61cda625a25049bca6942aabf3409ed Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Sep 2024 02:09:04 +0000 Subject: [PATCH 17/22] build: Add cross test --- build/checks/default.nix | 12 ++++++++++++ build/lib/renderers.nix | 2 +- build/packages.nix | 25 +++++++++++++++---------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/build/checks/default.nix b/build/checks/default.nix index f13e6c1..67d6d70 100644 --- a/build/checks/default.nix +++ b/build/checks/default.nix @@ -39,6 +39,18 @@ in touch $out ''; + make-venv-cross = + let + pkgs' = pkgs.pkgsCross.aarch64-multiplatform; + python = pkgs'.python312; + crossSet = pkgs'.callPackage pyproject-nix.build.packages { + inherit python; + }; + in + crossSet.pythonPkgsHostHost.mkVirtualEnv "cross-venv" { + build = [ ]; + }; + prebuilt-wheel = pythonSet.pythonPkgsHostHost.callPackage ( { stdenv, diff --git a/build/lib/renderers.nix b/build/lib/renderers.nix index 2757d5d..27de53c 100644 --- a/build/lib/renderers.nix +++ b/build/lib/renderers.nix @@ -102,7 +102,7 @@ in assert assertMsg (!hasPrefix storeDir root) '' Editable root was passed as a Nix store path string. - ${optionalString (inPureEvalMode) '' + ${optionalString inPureEvalMode '' This is most likely because you are using Flakes, and are automatically inferring the editable root from projectRoot. Flakes are copied to the Nix store on evaluation. This can temporarily be worked around using --impure. ''} diff --git a/build/packages.nix b/build/packages.nix index c240f93..f2d99b9 100644 --- a/build/packages.nix +++ b/build/packages.nix @@ -46,6 +46,7 @@ in newScope, buildPackages, stdenv, + pkgs, }: makeScope newScope ( final: @@ -111,6 +112,17 @@ makeScope newScope ( hooks = bootstrapHooks; }; + isCross = stdenv.buildPlatform != stdenv.hostPlatform; + + # Python packages for the build host + pythonPkgsBuildHost = mkPythonSet { + inherit (if isCross then buildPackages else pkgs) stdenv newScope; + python = python.pythonOnBuildForHost; + inherit (final) pythonPkgsBuildHost; + bootstrapHooks = final.pythonPkgsBootstrap.hooks; + pythonPkgsFun = pkgsFun; + }; + in { # Allows overriding Python by calling overrideScope on the outer scope @@ -129,19 +141,12 @@ makeScope newScope ( }; }; - # Python packages for the build host - pythonPkgsBuildHost = mkPythonSet { - inherit (buildPackages) stdenv newScope; - python = python.pythonOnBuildForHost; - inherit (final) pythonPkgsBuildHost; - bootstrapHooks = final.pythonPkgsBootstrap.hooks; - pythonPkgsFun = pkgsFun; - }; + inherit pythonPkgsBuildHost; # Python packages for the target host pythonPkgsHostHost = # If we're not doing cross reference build host packages - if stdenv.buildPlatform != stdenv.hostPlatform then + if isCross then mkPythonSet { inherit (final) newScope pythonPkgsBuildHost; inherit python stdenv; @@ -149,6 +154,6 @@ makeScope newScope ( pythonPkgsFun = pkgsFun; } else - final.pythonPkgsBuildHost; + pythonPkgsBuildHost; } ) From 436246a8108dff0ccfaec7ad8d5d4af3ab33bb4f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Sep 2024 03:23:34 +0000 Subject: [PATCH 18/22] Bump flake inputs --- flake.lock | 12 ++++++------ ...yproject.testProjectRenderBuildPythonPackage.json | 2 +- .../project.loadPyprojectDynamic.testPep621.json | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index ad5be22..91334c8 100644 --- a/flake.lock +++ b/flake.lock @@ -96,11 +96,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1725103162, - "narHash": "sha256-Ym04C5+qovuQDYL/rKWSR+WESseQBbNAe5DsXNx5trY=", + "lastModified": 1726937504, + "narHash": "sha256-bvGoiQBvponpZh8ClUcmJ6QnsNKw0EMrCQJARK3bI1c=", "owner": "nixos", "repo": "nixpkgs", - "rev": "12228ff1752d7b7624a54e9c1af4b222b3c1073b", + "rev": "9357f4f23713673f310988025d9dc261c20e70c6", "type": "github" }, "original": { @@ -126,11 +126,11 @@ ] }, "locked": { - "lastModified": 1724833132, - "narHash": "sha256-F4djBvyNRAXGusJiNYInqR6zIMI3rvlp6WiKwsRISos=", + "lastModified": 1727098951, + "narHash": "sha256-gplorAc0ISAUPemUNOnRUs7jr3WiLiHZb3DJh++IkZs=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "3ffd842a5f50f435d3e603312eefa4790db46af5", + "rev": "35dfece10c642eb52928a48bee7ac06a59f93e9a", "type": "github" }, "original": { diff --git a/lib/expected/project.loadPoetryPyproject.testProjectRenderBuildPythonPackage.json b/lib/expected/project.loadPoetryPyproject.testProjectRenderBuildPythonPackage.json index 8f7c538..049a080 100644 --- a/lib/expected/project.loadPoetryPyproject.testProjectRenderBuildPythonPackage.json +++ b/lib/expected/project.loadPoetryPyproject.testProjectRenderBuildPythonPackage.json @@ -206,7 +206,7 @@ }, { "pname": "types-requests", - "version": "2.32.0.20240712" + "version": "2.32.0.20240914" }, { "pname": "typing-extensions", diff --git a/lib/expected/project.loadPyprojectDynamic.testPep621.json b/lib/expected/project.loadPyprojectDynamic.testPep621.json index 6831e5e..027a188 100644 --- a/lib/expected/project.loadPyprojectDynamic.testPep621.json +++ b/lib/expected/project.loadPyprojectDynamic.testPep621.json @@ -131,7 +131,7 @@ }, { "pname": "pyreadstat", - "version": "1.2.6" + "version": "1.2.7" }, { "pname": "pytest", @@ -357,7 +357,7 @@ "spss": [ { "pname": "pyreadstat", - "version": "1.2.6" + "version": "1.2.7" } ], "sql-other": [ From 829f2a22db5afe64ce08a79776311546f3c511e8 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Sep 2024 03:48:54 +0000 Subject: [PATCH 19/22] doc: Add builders to use-cases --- doc/src/SUMMARY.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 29b5ead..84a1160 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -4,8 +4,14 @@ # Use cases -- [pyproject.toml](./use-cases/pyproject.md) -- [requirements.txt](./use-cases/requirements.md) +- [Developing with nixpkgs]() + - [pyproject.toml](./use-cases/pyproject.md) + - [requirements.txt](./use-cases/requirements.md) + +- [Python2nix]() + - [Builders](./build.md) + +# Meta - [FAQ](./FAQ.md) # Library reference From df00ec89a37d8335972d9b6afe36ab3d559a9eb7 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Sep 2024 04:33:33 +0000 Subject: [PATCH 20/22] treefmt: Enable shfmt --- build/hooks/pyproject-bootstrap-build-hook.sh | 18 ++++++------- build/hooks/pyproject-build-hook.sh | 16 ++++++------ build/hooks/pyproject-bytecode-hook.sh | 26 +++++++++---------- build/hooks/pyproject-configure-hook.sh | 24 ++++++++--------- build/hooks/pyproject-install-hook.sh | 26 +++++++++---------- build/hooks/pyproject-make-venv-hook.sh | 14 +++++----- build/hooks/pyproject-output-setup-hook.sh | 8 +++--- build/hooks/pyproject-pypa-install-hook.sh | 26 +++++++++---------- build/hooks/pyproject-wheel-dist-hook.sh | 20 +++++++------- dev/treefmt.nix | 3 +++ 10 files changed, 92 insertions(+), 89 deletions(-) diff --git a/build/hooks/pyproject-bootstrap-build-hook.sh b/build/hooks/pyproject-bootstrap-build-hook.sh index d3ac37c..a08d2a7 100644 --- a/build/hooks/pyproject-bootstrap-build-hook.sh +++ b/build/hooks/pyproject-bootstrap-build-hook.sh @@ -2,18 +2,18 @@ echo "Sourcing pyproject-build-hook" pyprojectBuildPhase() { - echo "Executing pyprojectBuildPhase" - runHook preBuild + echo "Executing pyprojectBuildPhase" + runHook preBuild - echo "Creating a wheel..." - @build@/bin/pyproject-build --no-isolation --outdir dist/ --wheel $pyprojectBuildFlags - echo "Finished creating a wheel..." + echo "Creating a wheel..." + @build@/bin/pyproject-build --no-isolation --outdir dist/ --wheel $pyprojectBuildFlags + echo "Finished creating a wheel..." - runHook postBuild - echo "Finished executing pyprojectBuildPhase" + runHook postBuild + echo "Finished executing pyprojectBuildPhase" } if [ -z "${dontUsePyprojectBuild-}" ] && [ -z "${buildPhase-}" ]; then - echo "Using pyprojectBuildPhase" - buildPhase=pyprojectBuildPhase + echo "Using pyprojectBuildPhase" + buildPhase=pyprojectBuildPhase fi diff --git a/build/hooks/pyproject-build-hook.sh b/build/hooks/pyproject-build-hook.sh index f8f4ad3..eef637d 100644 --- a/build/hooks/pyproject-build-hook.sh +++ b/build/hooks/pyproject-build-hook.sh @@ -2,17 +2,17 @@ echo "Sourcing pyproject-build-hook" pyprojectBuildPhase() { - echo "Executing pyprojectBuildPhase" - runHook preBuild + echo "Executing pyprojectBuildPhase" + runHook preBuild - echo "Creating a wheel..." - env PYTHONPATH="${NIX_PYPROJECT_PYTHONPATH}:${PYTHONPATH}" @build@/bin/pyproject-build --no-isolation --outdir dist/ --wheel $pypaBuildFlags + echo "Creating a wheel..." + env PYTHONPATH="${NIX_PYPROJECT_PYTHONPATH}:${PYTHONPATH}" @build@/bin/pyproject-build --no-isolation --outdir dist/ --wheel $pypaBuildFlags - runHook postBuild - echo "Finished executing pyprojectBuildPhase" + runHook postBuild + echo "Finished executing pyprojectBuildPhase" } if [ -z "${dontUsePyprojectBuild-}" ] && [ -z "${buildPhase-}" ]; then - echo "Using pyprojectBuildPhase" - buildPhase=pyprojectBuildPhase + echo "Using pyprojectBuildPhase" + buildPhase=pyprojectBuildPhase fi diff --git a/build/hooks/pyproject-bytecode-hook.sh b/build/hooks/pyproject-bytecode-hook.sh index 48459e8..637fd73 100644 --- a/build/hooks/pyproject-bytecode-hook.sh +++ b/build/hooks/pyproject-bytecode-hook.sh @@ -1,19 +1,19 @@ -pyprojectBytecodePhase () { - if [ -d "$out/bin" ]; then - rm -rf "$out/bin/__pycache__" # Python 3 - find "$out/bin" -type f -name "*.pyc" -delete # Python 2 - fi +pyprojectBytecodePhase() { + if [ -d "$out/bin" ]; then + rm -rf "$out/bin/__pycache__" # Python 3 + find "$out/bin" -type f -name "*.pyc" -delete # Python 2 + fi - items="$(find "$out" -name "@bytecodeName@")" - if [[ -n $items ]]; then - for pycache in $items; do - rm -rf "$pycache" - done - fi + items="$(find "$out" -name "@bytecodeName@")" + if [[ -n $items ]]; then + for pycache in $items; do + rm -rf "$pycache" + done + fi - @pythonInterpreter@ -OO -m compileall @compileArgs@ "$out"/@pythonSitePackages@ + @pythonInterpreter@ -OO -m compileall @compileArgs@ "$out"/@pythonSitePackages@ } if [ -z "${dontUsePyprojectBytecode-}" ]; then - postPhases+=" pyprojectBytecodePhase" + postPhases+=" pyprojectBytecodePhase" fi diff --git a/build/hooks/pyproject-configure-hook.sh b/build/hooks/pyproject-configure-hook.sh index 86f2775..4de4b91 100644 --- a/build/hooks/pyproject-configure-hook.sh +++ b/build/hooks/pyproject-configure-hook.sh @@ -2,21 +2,21 @@ echo "Sourcing pyproject-configure-hook" pyprojectConfigurePhase() { - echo "Executing pyprojectConfigurePhase" - runHook preConfigure + echo "Executing pyprojectConfigurePhase" + runHook preConfigure - # Undo any Python dependency propagation leaking into build, and set it to our interpreters PYTHONPATH - # - # In case of cross compilation this variable will contain two entries: - # One for the native Python and one for the cross built, so the native can load sysconfig - # information from the cross compiled Python. - export PYTHONPATH=@pythonPath@ + # Undo any Python dependency propagation leaking into build, and set it to our interpreters PYTHONPATH + # + # In case of cross compilation this variable will contain two entries: + # One for the native Python and one for the cross built, so the native can load sysconfig + # information from the cross compiled Python. + export PYTHONPATH=@pythonPath@ - runHook postConfigure - echo "Finished executing pyprojectConfigurePhase" + runHook postConfigure + echo "Finished executing pyprojectConfigurePhase" } if [ -z "${dontUsePyprojectConfigure-}" ] && [ -z "${configurePhase-}" ]; then - echo "Using pyprojectConfiguredPhase" - configurePhase=pyprojectConfigurePhase + echo "Using pyprojectConfiguredPhase" + configurePhase=pyprojectConfigurePhase fi diff --git a/build/hooks/pyproject-install-hook.sh b/build/hooks/pyproject-install-hook.sh index 599d063..d678f3b 100644 --- a/build/hooks/pyproject-install-hook.sh +++ b/build/hooks/pyproject-install-hook.sh @@ -2,25 +2,25 @@ echo "Sourcing pyproject-install-hook" pyprojectInstallPhase() { - echo "Executing pyprojectInstallPhase" - runHook preInstall + echo "Executing pyprojectInstallPhase" + runHook preInstall - pushd dist > /dev/null + pushd dist >/dev/null - for wheel in ./*.whl; do - @uv@/bin/uv pip --offline --no-cache install --no-deps --link-mode=copy --system --prefix "$out" $uvPipInstallFlags "$wheel" - echo "Successfully installed $wheel" - done + for wheel in ./*.whl; do + @uv@/bin/uv pip --offline --no-cache install --no-deps --link-mode=copy --system --prefix "$out" $uvPipInstallFlags "$wheel" + echo "Successfully installed $wheel" + done - rm -f "$out/.lock" + rm -f "$out/.lock" - popd > /dev/null + popd >/dev/null - runHook postInstall - echo "Finished executing pyprojectInstallPhase" + runHook postInstall + echo "Finished executing pyprojectInstallPhase" } if [ -z "${dontUsePyprojectInstall-}" ] && [ -z "${installPhase-}" ]; then - echo "Using pyprojectInstallPhase" - installPhase=pyprojectInstallPhase + echo "Using pyprojectInstallPhase" + installPhase=pyprojectInstallPhase fi diff --git a/build/hooks/pyproject-make-venv-hook.sh b/build/hooks/pyproject-make-venv-hook.sh index 1b3952b..b3dc89d 100644 --- a/build/hooks/pyproject-make-venv-hook.sh +++ b/build/hooks/pyproject-make-venv-hook.sh @@ -1,16 +1,16 @@ echo "Sourcing pyproject-make-venv-hook" pyprojectMakeVenv() { - echo "Executing pyprojectMakeVenv" - runHook preInstall + echo "Executing pyprojectMakeVenv" + runHook preInstall - @pythonInterpreter@ @makeVenvScript@ --python @python@ "$out" --env "NIX_PYPROJECT_DEPS" + @pythonInterpreter@ @makeVenvScript@ --python @python@ "$out" --env "NIX_PYPROJECT_DEPS" - runHook postInstall - echo "Finished executing pyprojectMakeVenv" + runHook postInstall + echo "Finished executing pyprojectMakeVenv" } if [ -z "${dontUsePyprojectMakeVenv-}" ] && [ -z "${installPhase-}" ]; then - echo "Using pyprojectMakeVenv" - installPhase=pyprojectMakeVenv + echo "Using pyprojectMakeVenv" + installPhase=pyprojectMakeVenv fi diff --git a/build/hooks/pyproject-output-setup-hook.sh b/build/hooks/pyproject-output-setup-hook.sh index 28a324c..d2ddafe 100644 --- a/build/hooks/pyproject-output-setup-hook.sh +++ b/build/hooks/pyproject-output-setup-hook.sh @@ -1,6 +1,6 @@ -pyprojectOutputSetupHook () { -mkdir -p $out/nix-support -cat >> $out/nix-support/setup-hook << EOF +pyprojectOutputSetupHook() { + mkdir -p $out/nix-support + cat >>$out/nix-support/setup-hook < /dev/null + pushd dist >/dev/null - for wheel in *.whl; do - env PYTHONPATH=$PYTHONPATH:@installer@/@pythonSitePackages@ @pythonInterpreter@ -m installer --prefix "$out" "$wheel" - echo "Successfully installed $wheel" - done + for wheel in *.whl; do + env PYTHONPATH=$PYTHONPATH:@installer@/@pythonSitePackages@ @pythonInterpreter@ -m installer --prefix "$out" "$wheel" + echo "Successfully installed $wheel" + done - rm -f "$out/.lock" + rm -f "$out/.lock" - popd > /dev/null + popd >/dev/null - runHook postInstall - echo "Finished executing pyprojectPypaInstallPhase" + runHook postInstall + echo "Finished executing pyprojectPypaInstallPhase" } if [ -z "${dontUsePyprojectInstall-}" ] && [ -z "${installPhase-}" ]; then - echo "Using pyprojectPypaInstallPhase" - installPhase=pyprojectPypaInstallPhase + echo "Using pyprojectPypaInstallPhase" + installPhase=pyprojectPypaInstallPhase fi diff --git a/build/hooks/pyproject-wheel-dist-hook.sh b/build/hooks/pyproject-wheel-dist-hook.sh index 932b193..a2e28f9 100644 --- a/build/hooks/pyproject-wheel-dist-hook.sh +++ b/build/hooks/pyproject-wheel-dist-hook.sh @@ -2,19 +2,19 @@ echo "Sourcing pyproject-wheel-dist-hook" pyprojectWheelDist() { - echo "Executing pyprojectWheelDist" - runHook preBuild + echo "Executing pyprojectWheelDist" + runHook preBuild - echo "Creating dist..." - mkdir -p dist - ln -s "$src" "dist/$(stripHash "$src")" + echo "Creating dist..." + mkdir -p dist + ln -s "$src" "dist/$(stripHash "$src")" - runHook postBuild - echo "Finished executing pyprojectWheelDist" + runHook postBuild + echo "Finished executing pyprojectWheelDist" } if [ -z "${dontUsePyprojectWheelDist-}" ] && [ -z "${buildPhase-}" ]; then - echo "Using pyprojectWheelDist" - buildPhase=pyprojectWheelDist - dontUnpack=1 + echo "Using pyprojectWheelDist" + buildPhase=pyprojectWheelDist + dontUnpack=1 fi diff --git a/dev/treefmt.nix b/dev/treefmt.nix index e70977e..9f7a53e 100644 --- a/dev/treefmt.nix +++ b/dev/treefmt.nix @@ -11,4 +11,7 @@ _: { programs.ruff-format.enable = true; programs.ruff-check.enable = true; programs.mypy.enable = true; + + # Shell + programs.shfmt.enable = true; } From 10f9e32be18988361bde4c7a8cb276a059d3db8c Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Sep 2024 07:28:58 +0000 Subject: [PATCH 21/22] doc: Add build usage doc --- doc/src/SUMMARY.md | 2 + doc/src/builders/minimal-lock.nix | 180 ++++++++++++++++++++++++++++++ doc/src/builders/packages.md | 15 +++ doc/src/builders/usage.md | 12 ++ 4 files changed, 209 insertions(+) create mode 100644 doc/src/builders/minimal-lock.nix create mode 100644 doc/src/builders/packages.md create mode 100644 doc/src/builders/usage.md diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 84a1160..767252f 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -10,6 +10,7 @@ - [Python2nix]() - [Builders](./build.md) + - [Usage](./builders/usage.md) # Meta - [FAQ](./FAQ.md) @@ -37,6 +38,7 @@ - [pip](./lib/pip.nix) - [Build](./build.md) + - [packages](./builders/packages.md) - [lib](./build/lib/default.nix) - [renderers](./build/lib/renderers.nix) - [resolvers](./build/lib/resolvers.nix) diff --git a/doc/src/builders/minimal-lock.nix b/doc/src/builders/minimal-lock.nix new file mode 100644 index 0000000..0c7f13c --- /dev/null +++ b/doc/src/builders/minimal-lock.nix @@ -0,0 +1,180 @@ +{ + python, + callPackage, + pyproject-nix, + lib, +}: +let + # Pyproject.nix packages quite a few, but not all build-system dependencies. + # + # We only package PyPI packages because lock file generators often miss this metadata, so it's required to help kickstart a Python set. + # This set is incomplete, and much smaller in both package count and scope than nixpkgs is. + baseSet = callPackage pyproject-nix.build.packages { + inherit python; + }; + + # A hypothetical over-simplified "lock file" format to demonstrate what typical usage would look like. + # This format is absent of important data such as PEP-508 markers and more. + # + # Also note that all sources are the same. In a real-world file these would of course all be different. + lock = + let + src = { + type = "pypi"; + url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz"; + hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"; + }; + in + { + package-a = { + name = "a"; + version = "1.0.0"; + dependencies = { }; + optional-dependencies = { }; + inherit src; + build-system = { + flit-core = [ ]; + }; + }; + + package-b = { + name = "b"; + version = "1.0.0"; + # Depend on a with no optionals + dependencies = { + a = [ ]; + }; + inherit src; + build-system = { + flit-core = [ ]; + }; + }; + + package-c = { + name = "c"; + version = "1.0.0"; + dependencies = { + a = [ ]; + }; + # Has an optional dependency on b when cool_feature is activated + optional-dependencies = { + cool_feature = { + b = [ ]; + }; + }; + inherit src; + build-system = { + flit-core = [ ]; + }; + }; + + package-d = { + name = "d"; + version = "1.0.0"; + dependencies = { + c = [ "cool_feature" ]; + }; + # A local package dependend on by it's path + src = { + type = "path"; + path = ./packages/d; + }; + build-system = { + flit-core = [ ]; + }; + }; + }; + + # Create a PEP-508 marker environment for marker evaluation + environ = pyproject-nix.lib.pep508.mkEnviron python; + + # Transform lock into a Pyproject.nix build overlay. + # This will create packages from the lock. + overlay = + pyfinal: _pyprev: + lib.mapAttrs ( + name: lockpkg: + # If package is a local package use a project loader from pyproject-nix.lib.project + if lockpkg.src.type == "path" then + ( + let + project = pyproject-nix.project.loadPyprojectDynamic { + projectRoot = lockpkg.src.path; + }; + in + pyfinal.callPackage ( + # Function called with callPackage + { + stdenv, + pyprojectHook, + resolveBuildSystem, + }: + # Call stdenv.mkDerivation with project + stdenv.mkDerivation ( + # Render stdenv.mkDerivation arguments from project + pyproject-nix.build.lib.renderers.mkDerivation + { + inherit project environ; + } + { + inherit pyprojectHook resolveBuildSystem; + } + ) + ) { } + ) + # If a package is a remote (pypi) package there is no ready made renderers to use. + # You need to apply your own transformations. + else if lockpkg.src.type == "pypi" then + pyfinal.callPackage ( + { + stdenv, + fetchurl, + pyprojectHook, + pyprojectBootstrapHook, + resolveBuildSystem, + }: + stdenv.mkDerivation { + pname = lockpkg.name; + inherit (lockpkg) version; + src = fetchurl lockpkg.src; + + nativeBuildInputs = + [ + # Check if package is a bootstrap package. If it is we should use pyprojectBootstrapHook. + (if pyproject-nix.build.lib.isBootstrapPackage name then pyprojectBootstrapHook else pyprojectHook) + ] + # Build systems needs to be resolved since we don't propagate dependencies. + # Otherwise dependencies of our build-system will be missing. + ++ resolveBuildSystem lockpkg.build-system; + + # Dependencies go in passthru to avoid polluting runtime package. + passthru = { + inherit (lockpkg) dependencies optional-dependencies; + }; + } + ) { } + else + throw "Unhandled src type: ${lockpkg.src.type}" null + ) lock; + + # Override set + pythonSet = baseSet.overrideScope ( + _final: _prev: { + # Override build platform dependencies + # + # Use this when overriding build-systems that need to run on the build platform. + pythonPkgsBuildHost = overlay; + + # Override target platform packages. + # + # Use this to override packages for the target platform. + pythonPkgsHostHost = overlay; + } + ); + +in +# Create a virtual environment containing our dependency specification +pythonSet.pythonPkgsHostHost.mkVirtualEnv "example-venv" { + # Depend on package + build = [ ]; +} diff --git a/doc/src/builders/packages.md b/doc/src/builders/packages.md new file mode 100644 index 0000000..cf588ad --- /dev/null +++ b/doc/src/builders/packages.md @@ -0,0 +1,15 @@ +# packages + +## Creating a base package set + +``` nix +# Returns a scope with base packages. + +pkgs.callPackage pyproject-nix.build.packages { + python = interpreter; +} +``` + +## Overriding scope + +See the [nixpkgs documentation](https://nixos.org/manual/nixpkgs/stable/#function-library-lib.customisation.makeScope). diff --git a/doc/src/builders/usage.md b/doc/src/builders/usage.md new file mode 100644 index 0000000..6691c46 --- /dev/null +++ b/doc/src/builders/usage.md @@ -0,0 +1,12 @@ +# Usage + +While the nixpkgs Python infrastructure is built mainly for manual packaging, `pyproject.nix`'s builders are mainly targeted at lock file consumers like [uv2nix](https://github.com/adisbladis/uv2nix). + +This example shows the essence of implementing a lock file converter in pure Nix using `pyproject.nix`. +A real world implementation is more complex. To see a lock file converter built according to `pyproject.nix` best practices see [uv2nix](https://github.com/adisbladis/uv2nix). + +## example.nix + +```nix +{{#include ./minimal-lock.nix}} +``` From 54300b9ed89fbc2dd477893f3ccc223eee3ed54f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 25 Sep 2024 07:55:17 +0000 Subject: [PATCH 22/22] doc: Add build hooks --- build/hooks/default.nix | 54 ++++++++++++++++++++++++++++++++++++++--- doc/src/SUMMARY.md | 1 + 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/build/hooks/default.nix b/build/hooks/default.nix index b6e6949..c4e0f75 100644 --- a/build/hooks/default.nix +++ b/build/hooks/default.nix @@ -17,7 +17,11 @@ let in { - # Build hook used to build PEP-621/setuptools projects + /* + Undo any `$PYTHONPATH` changes done by nixpkgs Python infrastructure dependency propagation. + + Used internally by `pyprojectHook`. + */ pyprojectConfigureHook = callPackage ( { python }: makeSetupHook { @@ -36,7 +40,11 @@ in } ./pyproject-configure-hook.sh ) { }; - # Build hook used to build PEP-621/setuptools projects + /* + Build a pyproject.toml/setuptools project. + + Used internally by `pyprojectHook`. + */ pyprojectBuildHook = callPackage ( _: makeSetupHook { @@ -51,6 +59,11 @@ in } ./pyproject-build-hook.sh ) { }; + /* + Symlink prebuilt wheel sources. + + Used internally by `pyprojectWheelHook`. + */ pyprojectWheelDistHook = callPackage ( _: makeSetupHook { @@ -58,6 +71,11 @@ in } ./pyproject-wheel-dist-hook.sh ) { }; + /* + Install built projects from dist/*.whl. + + Used internally by `pyprojectHook`. + */ pyprojectInstallHook = callPackage ( @@ -73,6 +91,11 @@ in inherit (buildPackages) uv; }; + /* + Install hook using pypa/installer. + + Used instead of `pyprojectInstallHook` for cross compilation support. + */ pyprojectPypaInstallHook = callPackage ( { pythonPkgsBuildHost }: makeSetupHook { @@ -84,6 +107,11 @@ in } ./pyproject-pypa-install-hook.sh ) { }; + /* + Clean up any shipped bytecode in package output and recompile. + + Used internally by `pyprojectHook`. + */ pyprojectBytecodeHook = callPackage ( _: makeSetupHook { @@ -103,6 +131,11 @@ in } ./pyproject-bytecode-hook.sh ) { }; + /* + Create `pyproject.nix` setup hook in package output. + + Used internally by `pyprojectHook`. + */ pyprojectOutputSetupHook = callPackage ( _: makeSetupHook { @@ -113,6 +146,11 @@ in } ./pyproject-output-setup-hook.sh ) { }; + /* + Create a virtual environment from buildInputs + + Used internally by `mkVirtualEnv`. + */ pyprojectMakeVenvHook = callPackage ( { python }: makeSetupHook { @@ -124,7 +162,12 @@ in } ./pyproject-make-venv-hook.sh ) { }; - # Meta hook aggregating the default pyproject.toml/setup.py install behaviour and adds Python + /* + Meta hook aggregating the default pyproject.toml/setup.py install behaviour and adds Python. + + This is the default choice for both pyproject.toml & setuptools projects. + */ + # pyprojectHook = callPackage ( @@ -157,6 +200,11 @@ in }) ); + /* + Hook used to build prebuilt wheels. + + Use instead of pyprojectHook. + */ pyprojectWheelHook = hooks.pyprojectHook.override { pyprojectBuildHook = hooks.pyprojectWheelDistHook; }; diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 767252f..d8498e1 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -42,6 +42,7 @@ - [lib](./build/lib/default.nix) - [renderers](./build/lib/renderers.nix) - [resolvers](./build/lib/resolvers.nix) + - [hooks](./build/hooks/default.nix) # Contributing