WIP: Python no runtime deps at build time

pkgs/applications/science/logic/klee/default.nix

@@ -85,8 +85,7 @@ llvmPackages.stdenv.mkDerivation rec {
     # Should appear BEFORE lit, since lit passes through python rather
     # than the python environment we make.
     kleePython
-    (lit.override { python = kleePython; })
-  ];
+  ] ++ python3.pkgs.requiredPythonModules python3.pkgs.lit;
 
   cmakeBuildType = if debug then "Debug" else if !debug && includeDebugInfo then "RelWithDebInfo" else "MinSizeRel";
 

pkgs/development/interpreters/python/hooks/default.nix

@@ -251,7 +251,10 @@ in {
   sphinxHook = callPackage ({ makePythonHook, installShellFiles }:
     makePythonHook {
       name = "python${python.pythonVersion}-sphinx-hook";
-      propagatedBuildInputs = [ pythonOnBuildForHost.pkgs.sphinx installShellFiles ];
+      propagatedBuildInputs = [
+        pythonOnBuildForHost.pkgs.sphinx
+        installShellFiles
+      ] ++ (pythonOnBuildForHost.pkgs.requiredPythonModules [ pythonOnBuildForHost.pkgs.sphinx ]);
       substitutions = {
         sphinxBuild = "${pythonOnBuildForHost.pkgs.sphinx}/bin/sphinx-build";
       };

pkgs/development/interpreters/python/link.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+from functools import wraps
+from textwrap import dedent
+import os.path
+import stat
+import os
+import sys
+
+
+src = sys.argv[1]
+dst = sys.argv[2]
+
+
+def write_bin_wrapper(bin_name):
+    """Write a bin wrapper with NIX_PYTHONPATH set"""
+    src_bin = os.path.join(src, "bin", bin_name)
+    dst_bin = os.path.join(dst, "bin", bin_name)
+
+    # TODO: Non-executable files
+
+    script = "#!/usr/bin/env bash" + dedent("""
+    export NIX_PYTHONPATH="%s"
+    exec %s "$@"
+    """ % (os.environ["PYTHONPATH"], src_bin))
+
+    with open(dst_bin, "w") as fd:
+        fd.write(script)
+
+        mode = os.fstat(fd.fileno()).st_mode
+        mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+        os.fchmod(fd.fileno(), stat.S_IMODE(mode))
+
+
+@wraps(os.mkdir)
+def mkdir(*args, **kwargs):
+    """Wraps os.mkdir but doesn't fail on existing file"""
+    try:
+        os.mkdir(*args, **kwargs)
+    except FileExistsError:
+        pass
+
+
+def symlink_tree(src: str, dst: str):
+    mkdir(dst)
+
+    for root, dirs, files in os.walk(src):
+        dst_dir = os.path.join(dst, root.removeprefix(src).removeprefix(os.path.sep))
+        mkdir(dst_dir)
+
+        for dir in dirs:
+            mkdir(os.path.join(dst_dir, dir))
+        for filename in files:
+            os.symlink(os.path.join(root, filename), os.path.join(dst_dir, filename))
+
+
+if __name__ == "__main__":
+    # If the store path is a simple file no special handling can be done.
+    st = os.lstat(src)
+    if not stat.S_ISDIR(st.st_mode):
+        os.symlink(src, dst)
+        exit(0)
+
+    os.mkdir(dst)
+
+    for filename in os.listdir(src):
+        src_file = os.path.join(src, filename)
+        dst_file = os.path.join(dst, filename)
+
+        # Wrap bin's in a wrapper that sets NIX_PYTHONPATH.
+        if filename == "bin":
+            os.mkdir(dst_file)
+            for bin_name in os.listdir(src_file):
+                write_bin_wrapper(bin_name)
+            continue
+
+        st = os.lstat(src_file)
+        if stat.S_ISDIR(st.st_mode):
+            symlink_tree(src_file, dst_file)
+            continue
+
+        # For all other files we place a symlink to the original Python derivation.
+        os.symlink(src_file, dst_file)

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -8,6 +8,7 @@
 , ensureNewerSourcesForZipFilesHook
 # Whether the derivation provides a Python module or not.
 , toPythonModule
+, requiredPythonModules
 , namePrefix
 , update-python-libraries
 , setuptools
@@ -233,6 +234,8 @@ let
       inherit build-system;
     };
 
+  nativeBuildInputs' = nativeBuildInputs ++ build-system;
+
   # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
   self = toPythonModule (stdenv.mkDerivation ((cleanAttrs attrs) // {
 
@@ -270,12 +273,14 @@ let
       else
         pypaBuildHook
     ) (
-      if isBootstrapPackage then
-        pythonRuntimeDepsCheckHook.override {
-          inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
-        }
-      else
-        pythonRuntimeDepsCheckHook
+      null
+      # TODO: Reimplement as a passthru.tests once https://github.com/NixOS/nixpkgs/pull/272177 is merged
+      # if isBootstrapPackage then
+      #   pythonRuntimeDepsCheckHook.override {
+      #     inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
+      #   }
+      # else
+      #   pythonRuntimeDepsCheckHook
     )] ++ optionals (format' == "wheel") [
       wheelUnpackHook
     ] ++ optionals (format' == "egg") [
@@ -287,19 +292,16 @@ let
         }
       else
         pypaInstallHook
-    )] ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
-      # This is a test, however, it should be ran independent of the checkPhase and checkInputs
-      pythonImportsCheckHook
-    ] ++ optionals (python.pythonAtLeast "3.3") [
+    )] ++ optionals (python.pythonAtLeast "3.3") [
       # Optionally enforce PEP420 for python3
       pythonNamespacesHook
     ] ++ optionals withDistOutput [
       pythonOutputDistHook
-    ] ++ nativeBuildInputs ++ build-system;
+    ] ++ nativeBuildInputs' ++ requiredPythonModules nativeBuildInputs';
 
     buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
 
-    propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [
+    propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [
       # we propagate python even for packages transformed with 'toPythonApplication'
       # this pollutes the PATH but avoids rebuilds
       # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
@@ -312,7 +314,7 @@ let
 
     # Python packages don't have a checkPhase, only an installCheckPhase
     doCheck = false;
-    doInstallCheck = attrs.doCheck or true;
+    doInstallCheck = false; #attrs.doCheck or true;
     nativeInstallCheckInputs = [
     ] ++ optionals (format' == "setuptools") [
       # Longer-term we should get rid of this and require

pkgs/development/interpreters/python/python-packages-base.nix

@@ -8,6 +8,7 @@ self:
 
 let
   inherit (self) callPackage;
+  inherit (lib) concatMap;
 
   namePrefix = python.libPrefix + "-";
 
@@ -42,17 +43,14 @@ let
     inherit toPythonModule; # Libraries provide modules
   }));
 
-  buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage mkPythonDerivation {
-    namePrefix = "";        # Python applications should not have any prefix
-    toPythonModule = x: x;  # Application does not provide modules.
-  }));
+  buildPythonApplication = args: toPythonApplication (buildPythonPackage args);
 
   # Check whether a derivation provides a Python module.
   hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python;
 
   # Get list of required Python modules given a list of derivations.
   requiredPythonModules = drvs: let
-    modules = lib.filter hasPythonModule drvs;
+    modules = lib.filter hasPythonModule (if lib.isAttrs drvs then [ drvs ] else drvs);
   in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules));
 
   # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
@@ -63,29 +61,67 @@ let
 
   # Convert derivation to a Python module.
   toPythonModule = drv:
-    drv.overrideAttrs( oldAttrs: {
+    (drv.pythonPackage or drv).overrideAttrs( oldAttrs: {
       # Use passthru in order to prevent rebuilds when possible.
       passthru = (oldAttrs.passthru or {})// {
         pythonModule = python;
         pythonPath = [ ]; # Deprecated, for compatibility.
         requiredPythonModules =
           builtins.addErrorContext
           "while calculating requiredPythonModules for ${drv.name or drv.pname}:"
-          (requiredPythonModules drv.propagatedBuildInputs);
+          (requiredPythonModules (drv.propagatedBuildInputs ++ drv.passthru.dependencies or []));
       };
     });
 
   # Convert a Python library to an application.
-  toPythonApplication = drv:
-    drv.overrideAttrs( oldAttrs: {
-      passthru = (oldAttrs.passthru or {}) // {
-        # Remove Python prefix from name so we have a "normal" name.
-        # While the prefix shows up in the store path, it won't be
-        # used by `nix-env`.
-        name = removePythonPrefix oldAttrs.name;
-        pythonModule = false;
+  toPythonApplication = drv':
+    lib.makeOverridable ({
+      optional-dependencies ? [ ],
+    }:
+    let
+      drv = drv'.overrideAttrs( oldAttrs: {
+        passthru = (oldAttrs.passthru or {}) // {
+          # Remove Python prefix from name so we have a "normal" name.
+          # While the prefix shows up in the store path, it won't be
+          # used by `nix-env`.
+          name = removePythonPrefix oldAttrs.name;
+          pythonModule = false;
+        };
+      });
+
+      outputs = drv.outputs or [ "out" ];
+    in stdenv.mkDerivation {
+      inherit (drv) name version;
+      inherit outputs;
+
+      nativeBuildInputs = [ python ];
+      buildInputs = requiredPythonModules (
+        [ drv' ]
+        ++ concatMap (group: drv.passthru.optional-dependencies.${group}) optional-dependencies
+      );
+
+      dontUnpack = true;
+      dontConfigure = true;
+      dontBuild = true;
+
+      installPhase = ''
+        runHook preInstall
+        ${lib.concatStringsSep "\n" (map (output: "python3 ${./link.py} ${drv.${output}} \$${output}") outputs)}
+        runHook postInstall
+       '';
+
+      passthru = drv.passthru // {
+        pythonPackage = drv';
+      } // lib.optionalAttrs (drv ? src) {
+        inherit (drv) src;
       };
-    });
+
+      inherit (drv) meta;
+    } // lib.optionalAttrs (drv ? pname) {
+      inherit (drv) pname;
+    } // lib.optionalAttrs (drv ? version) {
+      inherit (drv) version;
+    }) { };
 
   disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}";
 

pkgs/development/interpreters/python/with-packages.nix

@@ -1,3 +1,9 @@
 { buildEnv, pythonPackages }:
 
-f: let packages = f pythonPackages; in buildEnv.override { extraLibs = packages; }
+f: let
+  packages = f pythonPackages;
+in
+buildEnv.override {
+  # Compute required dependencies recursively.
+  extraLibs = packages ++ pythonPackages.requiredPythonModules packages;
+}

pkgs/development/tools/documentation/gtk-doc/default.nix

@@ -60,7 +60,7 @@ python3.pkgs.buildPythonApplication rec {
     dblatex
   ];
 
-  pythonPath = with python3.pkgs; [
+  pythonPath = with python3.pkgs; requiredPythonModules [
     pygments # Needed for https://gitlab.gnome.org/GNOME/gtk-doc/blob/GTK_DOC_1_32/meson.build#L42
     lxml
   ];

pkgs/tools/inputmethods/input-remapper/default.nix

@@ -51,6 +51,9 @@ in
   '' + lib.optionalString withDebugLogLevel ''
     # if debugging
     substituteInPlace inputremapper/logger.py --replace "logger.setLevel(logging.INFO)"  "logger.setLevel(logging.DEBUG)"
+  '' + ''
+    # set revision for --version output
+    echo "COMMIT_HASH = '${src.rev}'" > inputremapper/commit_hash.py
   '';
 
   doCheck = withDoCheck;
@@ -144,14 +147,4 @@ in
     maintainers = with maintainers; [ LunNova ];
     mainProgram = "input-remapper-gtk";
   };
-}).overrideAttrs (final: prev: {
-  # Set in an override as buildPythonApplication doesn't yet support
-  # the `final:` arg yet from #119942 'overlay style overridable recursive attributes'
-  # this ensures the rev matches the input src's rev after overriding
-  # See https://discourse.nixos.org/t/avoid-rec-expresions-in-nixpkgs/8293/7 for more
-  # discussion
-  postPatch = prev.postPatch or "" + ''
-    # set revision for --version output
-    echo "COMMIT_HASH = '${final.src.rev}'" > inputremapper/commit_hash.py
-  '';
 })

pkgs/top-level/all-packages.nix

@@ -4966,10 +4966,7 @@ with pkgs;
   dvc = with python3.pkgs; toPythonApplication dvc;
 
   dvc-with-remotes = dvc.override {
-    enableGoogle = true;
-    enableAWS = true;
-    enableAzure = true;
-    enableSSH = true;
+    optional-dependencies = [ "gs" "s3" "azure" "ssh" ];
   };
 
   dynamic-colors = callPackage ../tools/misc/dynamic-colors { };