From f2cc4a6b4fd9ef32b120b45c801eb188e6a42efc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Nov 2024 09:46:51 -0500 Subject: [PATCH] Clean up packaging a bit - Multiple choices of stdenv are handled more consistently, especially for the dev shells which were previously not done correctly. - Some stray nix code was moving into the `packaging` directory --- flake.nix | 75 +++++++++---------- {scripts => packaging}/binary-tarball.nix | 10 +-- packaging/hydra.nix | 47 ++++++------ .../installer/default.nix | 0 {scripts => packaging/installer}/install.in | 0 tests/installer/default.nix | 2 +- tests/nixos/default.nix | 10 +-- 7 files changed, 69 insertions(+), 75 deletions(-) rename {scripts => packaging}/binary-tarball.nix (85%) rename scripts/installer.nix => packaging/installer/default.nix (100%) rename {scripts => packaging/installer}/install.in (100%) diff --git a/flake.nix b/flake.nix index cfb9021d50fc..dc6159640297 100644 --- a/flake.nix +++ b/flake.nix @@ -81,38 +81,32 @@ imports = [ ./maintainers/flake-module.nix ]; systems = lib.subtractLists crossSystems systems; perSystem = { system, ... }: { - _module.args.pkgs = nixpkgsFor.${system}.native; + _module.args.pkgs = nixpkgsFor.${system}.native.stdenvPackages; }; }; # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let - make-pkgs = crossSystem: stdenv: import nixpkgs { - localSystem = { - inherit system; - }; - crossSystem = if crossSystem == null then null else { - config = crossSystem; - } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { - useLLVM = true; - }; - overlays = [ - (overlayFor (p: p.${stdenv})) - ]; - }; - stdenvs = forAllStdenvs (make-pkgs null); - native = stdenvs.stdenvPackages; + make-pkgs = crossSystem: + forAllStdenvs (stdenv: import nixpkgs { + localSystem = { + inherit system; + }; + crossSystem = if crossSystem == null then null else { + config = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { + useLLVM = true; + }; + overlays = [ + (overlayFor (p: p.${stdenv})) + ]; + }); in { - inherit stdenvs native; - static = native.pkgsStatic; - cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + native = make-pkgs null; + cross = forAllCrossSystems make-pkgs; }); - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { - inherit nix; - }; - overlayFor = getStdenv: final: prev: let stdenv = getStdenv final; @@ -156,7 +150,6 @@ hydraJobs = import ./packaging/hydra.nix { inherit inputs - binaryTarball forAllCrossSystems forAllSystems lib @@ -172,11 +165,11 @@ installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = - let pkgs = nixpkgsFor.${system}.native; + let pkgs = nixpkgsFor.${system}.native.stdenvPackages; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; - repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; + repl-completion = nixpkgsFor.${system}.native.stdenvPackages.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { @@ -192,12 +185,12 @@ /* # Add "passthru" tests // flatMapAttrs ({ - "" = nixpkgsFor.${system}.native; - } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + "" = nixpkgsFor.${system}.native.stdenvPackages; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenvPackages.stdenv.hostPlatform.isDarwin) { # TODO: enable static builds for darwin, blocked on: # https://github.com/NixOS/nixpkgs/issues/320448 # TODO: disabled to speed up GHA CI. - #"static-" = nixpkgsFor.${system}.static; + #"static-" = nixpkgsFor.${system}.stdenvPackages.pkgsStatic; }) (nixpkgsPrefix: nixpkgs: flatMapAttrs nixpkgs.nixComponents @@ -218,12 +211,12 @@ packages = forAllSystems (system: { # Here we put attributes that map 1:1 into packages., ie # for which we don't apply the full build matrix such as cross or static. - inherit (nixpkgsFor.${system}.native) + inherit (nixpkgsFor.${system}.native.stdenvPackages) changelog-d; default = self.packages.${system}.nix-ng; - nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; - nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; - nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + nix-manual = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-manual; + nix-internal-api-docs = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-external-api-docs; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. // flatMapAttrs @@ -264,22 +257,22 @@ } (pkgName: { supportsCross ? true }: { # These attributes go right into `packages.`. - "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + "${pkgName}" = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.native.stdenvPackages.pkgsStatic.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. - "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages.nixComponents.${pkgName}; })) // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { # These attributes go right into `packages.`. - "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.native."${stdenvName}Packages".nixComponents.${pkgName}; }) ) // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let - pkgs = nixpkgsFor.${system}.native; + pkgs = nixpkgsFor.${system}.native.stdenvPackages; image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; }; in pkgs.runCommand @@ -299,14 +292,14 @@ in forAllSystems (system: prefixAttrs "native" (forAllStdenvs (stdenvName: makeShell { - pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages"; + pkgs = nixpkgsFor.${system}.native."${stdenvName}Packages"; })) // - lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) ( + lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenvPackages.stdenv.isDarwin) ( prefixAttrs "static" (forAllStdenvs (stdenvName: makeShell { pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic; })) // prefixAttrs "cross" (forAllCrossSystems (crossSystem: makeShell { - pkgs = nixpkgsFor.${system}.cross.${crossSystem}; + pkgs = nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages; })) ) // { diff --git a/scripts/binary-tarball.nix b/packaging/binary-tarball.nix similarity index 85% rename from scripts/binary-tarball.nix rename to packaging/binary-tarball.nix index 104189b0c818..abeced858b40 100644 --- a/scripts/binary-tarball.nix +++ b/packaging/binary-tarball.nix @@ -22,18 +22,18 @@ in runCommand "nix-binary-tarball-${version}" env '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ + cp ${../scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${../scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + substitute ${../scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + substitute ${../scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \ + substitute ${../scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 6da502079419..f54ddc745936 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -1,5 +1,4 @@ { inputs -, binaryTarball , forAllCrossSystems , forAllSystems , lib @@ -12,7 +11,7 @@ let inherit (inputs) nixpkgs nixpkgs-regression; installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + nixpkgsFor.x86_64-linux.native.stdenvPackages.callPackage ./installer { inherit tarballs; }; @@ -60,34 +59,34 @@ in { # Binary package for various platforms. build = forAllPackages (pkgName: - forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + forAllSystems (system: nixpkgsFor.${system}.native.stdenvPackages.nixComponents.${pkgName})); shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); buildStatic = forAllPackages (pkgName: - lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.native.stdenvPackages.pkgsStatic.nixComponents.${pkgName})); buildCross = forAllPackages (pkgName: # Hack to avoid non-evaling package (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id) (forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})))); + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages.nixComponents.${pkgName})))); buildNoGc = let components = forAllSystems (system: - nixpkgsFor.${system}.native.nixComponents.overrideScope (self: super: { + nixpkgsFor.${system}.native.stdenvPackages.nixComponents.overrideScope (self: super: { nix-expr = super.nix-expr.override { enableGC = false; }; }) ); in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-cli); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. buildReadlineNoMarkdown = let components = forAllSystems (system: - nixpkgsFor.${system}.native.nixComponents.overrideScope (self: super: { + nixpkgsFor.${system}.native.stdenvPackages.nixComponents.overrideScope (self: super: { nix-cmd = super.nix-cmd.override { enableMarkdown = false; readlineFlavor = "readline"; @@ -97,18 +96,17 @@ in in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + binaryTarball = forAllSystems (system: + nixpkgsFor.${system}.native.stdenvPackages.callPackage ./binary-tarball.nix {}); binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: forAllCrossSystems (crossSystem: - binaryTarball - nixpkgsFor.${system}.cross.${crossSystem}.nix - nixpkgsFor.${system}.cross.${crossSystem})); + nixpkgsFor.${system}.cross.${crossSystem}.stdenvPackages.callPackage ./binary-tarball.nix {})); # The first half of the installation script. This is uploaded # to https://nixos.org/nix/install. It downloads the binary @@ -140,28 +138,31 @@ in dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { + coverage = nixpkgsFor.x86_64-linux.native.stdenvPackages.nix.override { pname = "nix-coverage"; withCoverageChecks = true; }; # Nix's manual - manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual; + manual = nixpkgsFor.x86_64-linux.native.stdenvPackages.nixComponents.nix-manual; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + internal-api-docs = nixpkgsFor.x86_64-linux.native.stdenvPackages.nixComponents.nix-internal-api-docs; # API docs for Nix's C bindings. - external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + external-api-docs = nixpkgsFor.x86_64-linux.native.stdenvPackages.nixComponents.nix-external-api-docs; # System tests. - tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + tests = import ../tests/nixos { + inherit lib nixpkgs nixpkgsFor; + inherit (self.inputs) nixpkgs-23-11; + } // { # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. evalNixpkgs = let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + inherit (nixpkgsFor.x86_64-linux.native.stdenvPackages) runCommand nix; in runCommand "eval-nixos" { buildInputs = [ nix ]; } '' @@ -179,20 +180,20 @@ in forAllSystems (system: import (nixpkgs + "/lib/tests/test-with-nix.nix") { - lib = nixpkgsFor.${system}.native.lib; + lib = nixpkgsFor.${system}.native.stdenvPackages.lib; nix = self.packages.${system}.nix; - pkgs = nixpkgsFor.${system}.native; + pkgs = nixpkgsFor.${system}.native.stdenvPackages; } ); }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; + pkgs = nixpkgsFor.x86_64-linux.native.stdenvPackages; nixpkgs = nixpkgs-regression; }; installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in + let pkgs = nixpkgsFor.${system}.native.stdenvPackages; in pkgs.runCommand "install-tests" { againstSelf = testNixVersions pkgs pkgs.nix; diff --git a/scripts/installer.nix b/packaging/installer/default.nix similarity index 100% rename from scripts/installer.nix rename to packaging/installer/default.nix diff --git a/scripts/install.in b/packaging/installer/install.in similarity index 100% rename from scripts/install.in rename to packaging/installer/install.in diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 4aed6eae489f..bbca9e4c8aba 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -139,7 +139,7 @@ let makeTest = imageName: testName: let image = images.${imageName}; in - with nixpkgsFor.${image.system}.native; + with nixpkgsFor.${image.system}.native.stdenvPackages; runCommand "installer-test-${imageName}-${testName}" { buildInputs = [ qemu_kvm openssh ]; diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 17bfdea38224..8073245ed45a 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -1,4 +1,4 @@ -{ lib, nixpkgs, nixpkgsFor, self }: +{ lib, nixpkgs, nixpkgsFor, nixpkgs-23-11 }: let @@ -17,12 +17,12 @@ let test ]; - hostPkgs = nixpkgsFor.${system}.native; + hostPkgs = nixpkgsFor.${system}.native.stdenvPackages; defaults = { - nixpkgs.pkgs = nixpkgsFor.${system}.native; + nixpkgs.pkgs = nixpkgsFor.${system}.native.stdenvPackages; nix.checkAllErrors = false; # TODO: decide which packaging stage to use. `nix-cli` is efficient, but not the same as the user-facing `everything.nix` package (`default`). Perhaps a good compromise is `everything.nix` + `noTests` defined above? - nix.package = nixpkgsFor.${system}.native.nixComponents.nix-cli; + nix.package = nixpkgsFor.${system}.native.stdenvPackages.nixComponents.nix-cli; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; @@ -59,7 +59,7 @@ let otherNixes.nix_2_13.setNixPackage = { lib, pkgs, ... }: { imports = [ checkOverrideNixVersion ]; nix.package = lib.mkForce ( - self.inputs.nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_13.overrideAttrs (o: { + nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_13.overrideAttrs (o: { meta = o.meta // { knownVulnerabilities = []; }; }) );