diff --git a/pkgs/applications/science/logic/klee/default.nix b/pkgs/applications/science/logic/klee/default.nix index fb02c6d40b9dc..a5af46d4fed73 100644 --- a/pkgs/applications/science/logic/klee/default.nix +++ b/pkgs/applications/science/logic/klee/default.nix @@ -85,8 +85,7 @@ llvmPackages.stdenv.mkDerivation rec { # Should appear BEFORE lit, since lit passes through python rather # than the python environment we make. kleePython - (lit.override { python = kleePython; }) - ]; + ] ++ python3.pkgs.requiredPythonModules python3.pkgs.lit; cmakeBuildType = if debug then "Debug" else if !debug && includeDebugInfo then "RelWithDebInfo" else "MinSizeRel"; diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index c61cd77fc7ecd..8f5b732cef0ed 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -251,7 +251,10 @@ in { sphinxHook = callPackage ({ makePythonHook, installShellFiles }: makePythonHook { name = "python${python.pythonVersion}-sphinx-hook"; - propagatedBuildInputs = [ pythonOnBuildForHost.pkgs.sphinx installShellFiles ]; + propagatedBuildInputs = [ + pythonOnBuildForHost.pkgs.sphinx + installShellFiles + ] ++ (pythonOnBuildForHost.pkgs.requiredPythonModules [ pythonOnBuildForHost.pkgs.sphinx ]); substitutions = { sphinxBuild = "${pythonOnBuildForHost.pkgs.sphinx}/bin/sphinx-build"; }; diff --git a/pkgs/development/interpreters/python/link.py b/pkgs/development/interpreters/python/link.py new file mode 100644 index 0000000000000..c12cebce48d9c --- /dev/null +++ b/pkgs/development/interpreters/python/link.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +from functools import wraps +from textwrap import dedent +import os.path +import stat +import os +import sys + + +src = sys.argv[1] +dst = sys.argv[2] + + +def write_bin_wrapper(bin_name): + """Write a bin wrapper with NIX_PYTHONPATH set""" + src_bin = os.path.join(src, "bin", bin_name) + dst_bin = os.path.join(dst, "bin", bin_name) + + # TODO: Non-executable files + + script = "#!/usr/bin/env bash" + dedent(""" + export NIX_PYTHONPATH="%s" + exec %s "$@" + """ % (os.environ["PYTHONPATH"], src_bin)) + + with open(dst_bin, "w") as fd: + fd.write(script) + + mode = os.fstat(fd.fileno()).st_mode + mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + os.fchmod(fd.fileno(), stat.S_IMODE(mode)) + + +@wraps(os.mkdir) +def mkdir(*args, **kwargs): + """Wraps os.mkdir but doesn't fail on existing file""" + try: + os.mkdir(*args, **kwargs) + except FileExistsError: + pass + + +def symlink_tree(src: str, dst: str): + mkdir(dst) + + for root, dirs, files in os.walk(src): + dst_dir = os.path.join(dst, root.removeprefix(src).removeprefix(os.path.sep)) + mkdir(dst_dir) + + for dir in dirs: + mkdir(os.path.join(dst_dir, dir)) + for filename in files: + os.symlink(os.path.join(root, filename), os.path.join(dst_dir, filename)) + + +if __name__ == "__main__": + # If the store path is a simple file no special handling can be done. + st = os.lstat(src) + if not stat.S_ISDIR(st.st_mode): + os.symlink(src, dst) + exit(0) + + os.mkdir(dst) + + for filename in os.listdir(src): + src_file = os.path.join(src, filename) + dst_file = os.path.join(dst, filename) + + # Wrap bin's in a wrapper that sets NIX_PYTHONPATH. + if filename == "bin": + os.mkdir(dst_file) + for bin_name in os.listdir(src_file): + write_bin_wrapper(bin_name) + continue + + st = os.lstat(src_file) + if stat.S_ISDIR(st.st_mode): + symlink_tree(src_file, dst_file) + continue + + # For all other files we place a symlink to the original Python derivation. + os.symlink(src_file, dst_file) diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 0d6ab22249916..fc5fa45b11dc9 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -8,6 +8,7 @@ , ensureNewerSourcesForZipFilesHook # Whether the derivation provides a Python module or not. , toPythonModule +, requiredPythonModules , namePrefix , update-python-libraries , setuptools @@ -233,6 +234,8 @@ let inherit build-system; }; + nativeBuildInputs' = nativeBuildInputs ++ build-system; + # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. self = toPythonModule (stdenv.mkDerivation ((cleanAttrs attrs) // { @@ -270,12 +273,14 @@ let else pypaBuildHook ) ( - if isBootstrapPackage then - pythonRuntimeDepsCheckHook.override { - inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging; - } - else - pythonRuntimeDepsCheckHook + null + # TODO: Reimplement as a passthru.tests once https://github.com/NixOS/nixpkgs/pull/272177 is merged + # if isBootstrapPackage then + # pythonRuntimeDepsCheckHook.override { + # inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging; + # } + # else + # pythonRuntimeDepsCheckHook )] ++ optionals (format' == "wheel") [ wheelUnpackHook ] ++ optionals (format' == "egg") [ @@ -287,19 +292,16 @@ let } else pypaInstallHook - )] ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ - # This is a test, however, it should be ran independent of the checkPhase and checkInputs - pythonImportsCheckHook - ] ++ optionals (python.pythonAtLeast "3.3") [ + )] ++ optionals (python.pythonAtLeast "3.3") [ # Optionally enforce PEP420 for python3 pythonNamespacesHook ] ++ optionals withDistOutput [ pythonOutputDistHook - ] ++ nativeBuildInputs ++ build-system; + ] ++ nativeBuildInputs' ++ requiredPythonModules nativeBuildInputs'; buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); - propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [ + propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [ # we propagate python even for packages transformed with 'toPythonApplication' # this pollutes the PATH but avoids rebuilds # see https://github.com/NixOS/nixpkgs/issues/170887 for more context @@ -312,7 +314,7 @@ let # Python packages don't have a checkPhase, only an installCheckPhase doCheck = false; - doInstallCheck = attrs.doCheck or true; + doInstallCheck = false; #attrs.doCheck or true; nativeInstallCheckInputs = [ ] ++ optionals (format' == "setuptools") [ # Longer-term we should get rid of this and require diff --git a/pkgs/development/interpreters/python/python-packages-base.nix b/pkgs/development/interpreters/python/python-packages-base.nix index 2306292eb8c8f..d38b507443b9e 100644 --- a/pkgs/development/interpreters/python/python-packages-base.nix +++ b/pkgs/development/interpreters/python/python-packages-base.nix @@ -8,6 +8,7 @@ self: let inherit (self) callPackage; + inherit (lib) concatMap; namePrefix = python.libPrefix + "-"; @@ -42,17 +43,14 @@ let inherit toPythonModule; # Libraries provide modules })); - buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage mkPythonDerivation { - namePrefix = ""; # Python applications should not have any prefix - toPythonModule = x: x; # Application does not provide modules. - })); + buildPythonApplication = args: toPythonApplication (buildPythonPackage args); # Check whether a derivation provides a Python module. hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python; # Get list of required Python modules given a list of derivations. requiredPythonModules = drvs: let - modules = lib.filter hasPythonModule drvs; + modules = lib.filter hasPythonModule (if lib.isAttrs drvs then [ drvs ] else drvs); in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules)); # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations @@ -63,7 +61,7 @@ let # Convert derivation to a Python module. toPythonModule = drv: - drv.overrideAttrs( oldAttrs: { + (drv.pythonPackage or drv).overrideAttrs( oldAttrs: { # Use passthru in order to prevent rebuilds when possible. passthru = (oldAttrs.passthru or {})// { pythonModule = python; @@ -71,21 +69,59 @@ let requiredPythonModules = builtins.addErrorContext "while calculating requiredPythonModules for ${drv.name or drv.pname}:" - (requiredPythonModules drv.propagatedBuildInputs); + (requiredPythonModules (drv.propagatedBuildInputs ++ drv.passthru.dependencies or [])); }; }); # Convert a Python library to an application. - toPythonApplication = drv: - drv.overrideAttrs( oldAttrs: { - passthru = (oldAttrs.passthru or {}) // { - # Remove Python prefix from name so we have a "normal" name. - # While the prefix shows up in the store path, it won't be - # used by `nix-env`. - name = removePythonPrefix oldAttrs.name; - pythonModule = false; + toPythonApplication = drv': + lib.makeOverridable ({ + optional-dependencies ? [ ], + }: + let + drv = drv'.overrideAttrs( oldAttrs: { + passthru = (oldAttrs.passthru or {}) // { + # Remove Python prefix from name so we have a "normal" name. + # While the prefix shows up in the store path, it won't be + # used by `nix-env`. + name = removePythonPrefix oldAttrs.name; + pythonModule = false; + }; + }); + + outputs = drv.outputs or [ "out" ]; + in stdenv.mkDerivation { + inherit (drv) name version; + inherit outputs; + + nativeBuildInputs = [ python ]; + buildInputs = requiredPythonModules ( + [ drv' ] + ++ concatMap (group: drv.passthru.optional-dependencies.${group}) optional-dependencies + ); + + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + ${lib.concatStringsSep "\n" (map (output: "python3 ${./link.py} ${drv.${output}} \$${output}") outputs)} + runHook postInstall + ''; + + passthru = drv.passthru // { + pythonPackage = drv'; + } // lib.optionalAttrs (drv ? src) { + inherit (drv) src; }; - }); + + inherit (drv) meta; + } // lib.optionalAttrs (drv ? pname) { + inherit (drv) pname; + } // lib.optionalAttrs (drv ? version) { + inherit (drv) version; + }) { }; disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}"; diff --git a/pkgs/development/interpreters/python/with-packages.nix b/pkgs/development/interpreters/python/with-packages.nix index e1de0b2ee4cad..92ed5a22345ca 100644 --- a/pkgs/development/interpreters/python/with-packages.nix +++ b/pkgs/development/interpreters/python/with-packages.nix @@ -1,3 +1,9 @@ { buildEnv, pythonPackages }: -f: let packages = f pythonPackages; in buildEnv.override { extraLibs = packages; } +f: let + packages = f pythonPackages; +in +buildEnv.override { + # Compute required dependencies recursively. + extraLibs = packages ++ pythonPackages.requiredPythonModules packages; +} diff --git a/pkgs/development/tools/documentation/gtk-doc/default.nix b/pkgs/development/tools/documentation/gtk-doc/default.nix index b21bdfc500daa..086d9ef32703f 100644 --- a/pkgs/development/tools/documentation/gtk-doc/default.nix +++ b/pkgs/development/tools/documentation/gtk-doc/default.nix @@ -60,7 +60,7 @@ python3.pkgs.buildPythonApplication rec { dblatex ]; - pythonPath = with python3.pkgs; [ + pythonPath = with python3.pkgs; requiredPythonModules [ pygments # Needed for https://gitlab.gnome.org/GNOME/gtk-doc/blob/GTK_DOC_1_32/meson.build#L42 lxml ]; diff --git a/pkgs/tools/inputmethods/input-remapper/default.nix b/pkgs/tools/inputmethods/input-remapper/default.nix index c90eb780e7d64..e1eb48503339e 100644 --- a/pkgs/tools/inputmethods/input-remapper/default.nix +++ b/pkgs/tools/inputmethods/input-remapper/default.nix @@ -51,6 +51,9 @@ in '' + lib.optionalString withDebugLogLevel '' # if debugging substituteInPlace inputremapper/logger.py --replace "logger.setLevel(logging.INFO)" "logger.setLevel(logging.DEBUG)" + '' + '' + # set revision for --version output + echo "COMMIT_HASH = '${src.rev}'" > inputremapper/commit_hash.py ''; doCheck = withDoCheck; @@ -144,14 +147,4 @@ in maintainers = with maintainers; [ LunNova ]; mainProgram = "input-remapper-gtk"; }; -}).overrideAttrs (final: prev: { - # Set in an override as buildPythonApplication doesn't yet support - # the `final:` arg yet from #119942 'overlay style overridable recursive attributes' - # this ensures the rev matches the input src's rev after overriding - # See https://discourse.nixos.org/t/avoid-rec-expresions-in-nixpkgs/8293/7 for more - # discussion - postPatch = prev.postPatch or "" + '' - # set revision for --version output - echo "COMMIT_HASH = '${final.src.rev}'" > inputremapper/commit_hash.py - ''; }) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 913763e8c4239..eae34bf3e7857 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -4966,10 +4966,7 @@ with pkgs; dvc = with python3.pkgs; toPythonApplication dvc; dvc-with-remotes = dvc.override { - enableGoogle = true; - enableAWS = true; - enableAzure = true; - enableSSH = true; + optional-dependencies = [ "gs" "s3" "azure" "ssh" ]; }; dynamic-colors = callPackage ../tools/misc/dynamic-colors { };