Clean up packaging a bit

- Multiple choices of stdenv are handled more consistently, especially for the dev shells which were previously not done correctly. - Some stray nix code was moving into the `packaging` directory
obsidiansystems · Nov 4, 2024 · f2cc4a6 · f2cc4a6
1 parent 97bc114
commit f2cc4a6
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 75 deletions.
diff --git a/flake.nix b/flake.nix
@@ -81,38 +81,32 @@
         imports = [ ./maintainers/flake-module.nix ];
         systems = lib.subtractLists crossSystems systems;
         perSystem = { system, ... }: {
-          _module.args.pkgs = nixpkgsFor.${system}.native;
+          _module.args.pkgs = nixpkgsFor.${system}.native.stdenvPackages;
         };
       };
 
       # Memoize nixpkgs for different platforms for efficiency.
       nixpkgsFor = forAllSystems
         (system: let
-          make-pkgs = crossSystem: stdenv: import nixpkgs {
-            localSystem = {
-              inherit system;
-            };
-            crossSystem = if crossSystem == null then null else {
-              config = crossSystem;
-            } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
-              useLLVM = true;
-            };
-            overlays = [
-              (overlayFor (p: p.${stdenv}))
-            ];
-          };
-          stdenvs = forAllStdenvs (make-pkgs null);
-          native = stdenvs.stdenvPackages;
+          make-pkgs = crossSystem:
+            forAllStdenvs (stdenv: import nixpkgs {
+              localSystem = {
+                inherit system;
+              };
+              crossSystem = if crossSystem == null then null else {
+                config = crossSystem;
+              } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
+                useLLVM = true;
+              };
+              overlays = [
+                (overlayFor (p: p.${stdenv}))
+              ];
+            });
         in {
-          inherit stdenvs native;
-          static = native.pkgsStatic;
-          cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
+          native = make-pkgs null;
+          cross = forAllCrossSystems make-pkgs;
         });
 
-      binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
-        inherit nix;
-      };
-
       overlayFor = getStdenv: final: prev:
         let
           stdenv = getStdenv final;
@@ -156,7 +150,6 @@
       hydraJobs = import ./packaging/hydra.nix {
         inherit
           inputs
-          binaryTarball
           forAllCrossSystems
           forAllSystems
           lib
@@ -172,11 +165,11 @@
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
         rl-next =
-          let pkgs = nixpkgsFor.${system}.native;
+          let pkgs = nixpkgsFor.${system}.native.stdenvPackages;
           in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
           LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
         '';
-        repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { };
+        repl-completion = nixpkgsFor.${system}.native.stdenvPackages.callPackage ./tests/repl-completion.nix { };
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {
@@ -192,12 +185,12 @@
       /*
       # Add "passthru" tests
       // flatMapAttrs ({
-          "" = nixpkgsFor.${system}.native;
-        } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) {
+          "" = nixpkgsFor.${system}.native.stdenvPackages;
+        } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenvPackages.stdenv.hostPlatform.isDarwin) {
           # TODO: enable static builds for darwin, blocked on:
           #       https://github.com/NixOS/nixpkgs/issues/320448
           # TODO: disabled to speed up GHA CI.
-          #"static-" = nixpkgsFor.${system}.static;
+          #"static-" = nixpkgsFor.${system}.stdenvPackages.pkgsStatic;
         })
         (nixpkgsPrefix: nixpkgs:
           flatMapAttrs nixpkgs.nixComponents
@@ -218,12 +211,12 @@
       packages = forAllSystems (system:
         { # Here we put attributes that map 1:1 into packages.<system>, ie
           # for which we don't apply the full build matrix such as cross or static.
-          inherit (nixpkgsFor.${system}.native)
+          inherit (nixpkgsFor.${system}.native.stdenvPackages)
             changelog-d;
           default = self.packages.${system}.nix-ng;
-          nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual;
-          nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
-          nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
+          nix-manual = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-manual;
+          nix-internal-api-docs = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-internal-api-docs;
+          nix-external-api-docs = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-external-api-docs;
         }
         # We need to flatten recursive attribute sets of derivations to pass `flake check`.
         // flatMapAttrs
@@ -264,22 +257,22 @@
           }
           (pkgName: { supportsCross ? true }: {
               # These attributes go right into `packages.<system>`.
-              "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
-              "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
+              "${pkgName}" = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.${pkgName};
+              "${pkgName}-static" = nixpkgsFor.${system}.native.stdenvPackages.pkgsStatic.nixComponents.${pkgName};
             }
             // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
               # These attributes go right into `packages.<system>`.
-              "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
+              "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages.nixComponents.${pkgName};
             }))
             // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: {
               # These attributes go right into `packages.<system>`.
-              "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
+              "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.native."${stdenvName}Packages".nixComponents.${pkgName};
             })
           )
         // lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         dockerImage =
           let
-            pkgs = nixpkgsFor.${system}.native;
+            pkgs = nixpkgsFor.${system}.native.stdenvPackages;
             image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; };
           in
           pkgs.runCommand
@@ -299,14 +292,14 @@
       in
         forAllSystems (system:
           prefixAttrs "native" (forAllStdenvs (stdenvName: makeShell {
-            pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages";
+            pkgs = nixpkgsFor.${system}.native."${stdenvName}Packages";
           })) //
-          lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) (
+          lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenvPackages.stdenv.isDarwin) (
             prefixAttrs "static" (forAllStdenvs (stdenvName: makeShell {
               pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic;
             })) //
             prefixAttrs "cross" (forAllCrossSystems (crossSystem: makeShell {
-              pkgs = nixpkgsFor.${system}.cross.${crossSystem};
+              pkgs = nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages;
             }))
           ) //
           {

diff --git a/scripts/binary-tarball.nix → packaging/binary-tarball.nix b/scripts/binary-tarball.nix → packaging/binary-tarball.nix
@@ -22,18 +22,18 @@ in
 
 runCommand "nix-binary-tarball-${version}" env ''
   cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-  cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-  substitute ${./install-nix-from-closure.sh} $TMPDIR/install \
+  cp ${../scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+  substitute ${../scripts/install-nix-from-closure.sh} $TMPDIR/install \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
 
-  substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+  substitute ${../scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
-  substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+  substitute ${../scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
-  substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \
+  substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
     --subst-var-by nix ${nix} \
     --subst-var-by cacert ${cacert}
 

diff --git a/packaging/hydra.nix b/packaging/hydra.nix
@@ -1,5 +1,4 @@
 { inputs
-, binaryTarball
 , forAllCrossSystems
 , forAllSystems
 , lib
@@ -12,7 +11,7 @@ let
   inherit (inputs) nixpkgs nixpkgs-regression;
 
   installScriptFor = tarballs:
-    nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
+    nixpkgsFor.x86_64-linux.native.stdenvPackages.callPackage ./installer {
       inherit tarballs;
     };
 
@@ -60,34 +59,34 @@ in
 {
   # Binary package for various platforms.
   build = forAllPackages (pkgName:
-    forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}));
+    forAllSystems (system: nixpkgsFor.${system}.native.stdenvPackages.nixComponents.${pkgName}));
 
   shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
 
   buildStatic = forAllPackages (pkgName:
-    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName}));
+    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.native.stdenvPackages.pkgsStatic.nixComponents.${pkgName}));
 
   buildCross = forAllPackages (pkgName:
     # Hack to avoid non-evaling package
     (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id)
     (forAllCrossSystems (crossSystem:
-      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))));
+      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages.nixComponents.${pkgName}))));
 
   buildNoGc = let
     components = forAllSystems (system:
-      nixpkgsFor.${system}.native.nixComponents.overrideScope (self: super: {
+      nixpkgsFor.${system}.native.stdenvPackages.nixComponents.overrideScope (self: super: {
         nix-expr = super.nix-expr.override { enableGC = false; };
       })
     );
   in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
 
-  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli);
+  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-cli);
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.
   buildReadlineNoMarkdown = let
     components = forAllSystems (system:
-      nixpkgsFor.${system}.native.nixComponents.overrideScope (self: super: {
+      nixpkgsFor.${system}.native.stdenvPackages.nixComponents.overrideScope (self: super: {
         nix-cmd = super.nix-cmd.override {
           enableMarkdown = false;
           readlineFlavor = "readline";
@@ -97,18 +96,17 @@ in
   in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
 
   # Perl bindings for various platforms.
-  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings);
+  perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-perl-bindings);
 
   # Binary tarball for various platforms, containing a Nix store
   # with the closure of 'nix' package, and the second half of
   # the installation script.
-  binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
+  binaryTarball = forAllSystems (system:
+    nixpkgsFor.${system}.native.stdenvPackages.callPackage ./binary-tarball.nix {});
 
   binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system:
     forAllCrossSystems (crossSystem:
-      binaryTarball
-        nixpkgsFor.${system}.cross.${crossSystem}.nix
-        nixpkgsFor.${system}.cross.${crossSystem}));
+      nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages.callPackage ./binary-tarball.nix {}));
 
   # The first half of the installation script. This is uploaded
   # to https://nixos.org/nix/install. It downloads the binary
@@ -140,28 +138,31 @@ in
   dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
   # Line coverage analysis.
-  coverage = nixpkgsFor.x86_64-linux.native.nix.override {
+  coverage = nixpkgsFor.x86_64-linux.native.stdenvPackages.nix.override {
     pname = "nix-coverage";
     withCoverageChecks = true;
   };
 
   # Nix's manual
-  manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual;
+  manual = nixpkgsFor.x86_64-linux.native.stdenvPackages.nixComponents.nix-manual;
 
   # API docs for Nix's unstable internal C++ interfaces.
-  internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs;
+  internal-api-docs = nixpkgsFor.x86_64-linux.native.stdenvPackages.nixComponents.nix-internal-api-docs;
 
   # API docs for Nix's C bindings.
-  external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs;
+  external-api-docs = nixpkgsFor.x86_64-linux.native.stdenvPackages.nixComponents.nix-external-api-docs;
 
   # System tests.
-  tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // {
+  tests = import ../tests/nixos {
+    inherit lib nixpkgs nixpkgsFor;
+    inherit (self.inputs) nixpkgs-23-11;
+  } // {
 
     # Make sure that nix-env still produces the exact same result
     # on a particular version of Nixpkgs.
     evalNixpkgs =
       let
-        inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
+        inherit (nixpkgsFor.x86_64-linux.native.stdenvPackages) runCommand nix;
       in
       runCommand "eval-nixos" { buildInputs = [ nix ]; }
         ''
@@ -179,20 +180,20 @@ in
       forAllSystems (system:
         import (nixpkgs + "/lib/tests/test-with-nix.nix")
           {
-            lib = nixpkgsFor.${system}.native.lib;
+            lib = nixpkgsFor.${system}.native.stdenvPackages.lib;
             nix = self.packages.${system}.nix;
-            pkgs = nixpkgsFor.${system}.native;
+            pkgs = nixpkgsFor.${system}.native.stdenvPackages;
           }
       );
   };
 
   metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
-    pkgs = nixpkgsFor.x86_64-linux.native;
+    pkgs = nixpkgsFor.x86_64-linux.native.stdenvPackages;
     nixpkgs = nixpkgs-regression;
   };
 
   installTests = forAllSystems (system:
-    let pkgs = nixpkgsFor.${system}.native; in
+    let pkgs = nixpkgsFor.${system}.native.stdenvPackages; in
     pkgs.runCommand "install-tests"
       {
         againstSelf = testNixVersions pkgs pkgs.nix;

diff --git a/scripts/installer.nix → packaging/installer/default.nix b/scripts/installer.nix → packaging/installer/default.nix
diff --git a/scripts/install.in → packaging/installer/install.in b/scripts/install.in → packaging/installer/install.in
diff --git a/tests/installer/default.nix b/tests/installer/default.nix
@@ -139,7 +139,7 @@ let
 
   makeTest = imageName: testName:
     let image = images.${imageName}; in
-    with nixpkgsFor.${image.system}.native;
+    with nixpkgsFor.${image.system}.native.stdenvPackages;
     runCommand
       "installer-test-${imageName}-${testName}"
       { buildInputs = [ qemu_kvm openssh ];

diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix
@@ -1,4 +1,4 @@
-{ lib, nixpkgs, nixpkgsFor, self }:
+{ lib, nixpkgs, nixpkgsFor, nixpkgs-23-11 }:
 
 let
 
@@ -17,12 +17,12 @@ let
         test
       ];
 
-      hostPkgs = nixpkgsFor.${system}.native;
+      hostPkgs = nixpkgsFor.${system}.native.stdenvPackages;
       defaults = {
-        nixpkgs.pkgs = nixpkgsFor.${system}.native;
+        nixpkgs.pkgs = nixpkgsFor.${system}.native.stdenvPackages;
         nix.checkAllErrors = false;
         # TODO: decide which packaging stage to use. `nix-cli` is efficient, but not the same as the user-facing `everything.nix` package (`default`). Perhaps a good compromise is `everything.nix` + `noTests` defined above?
-        nix.package = nixpkgsFor.${system}.native.nixComponents.nix-cli;
+        nix.package = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-cli;
       };
       _module.args.nixpkgs = nixpkgs;
       _module.args.system = system;
@@ -59,7 +59,7 @@ let
   otherNixes.nix_2_13.setNixPackage = { lib, pkgs, ... }: {
     imports = [ checkOverrideNixVersion ];
     nix.package = lib.mkForce (
-      self.inputs.nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_13.overrideAttrs (o: {
+      nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_13.overrideAttrs (o: {
         meta = o.meta // { knownVulnerabilities = []; };
       })
     );