Merge branch 'buildkite-nixprof-bak' into buildkite-nixprof

.github/workflows/pipeline.yml

@@ -14,8 +14,10 @@ defaults:
     shell: bash
 
 # do we really want nixos-unstable?
-env: 
+env:
   NIX_PATH: "nixpkgs=channel:nixos-unstable"
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  PR_NUMBER: ${{ github.event.number }}
 
 jobs:
   nix-build:
@@ -123,6 +125,13 @@ jobs:
       - name: "Run tests with ghc8107: End-2-end test of hix project initialization and flakes development shell ..."
         run: "./test/tests.sh ghc8107 hix"
 
+#  template:
+#    runs-on: [self-hosted, linux]
+#    steps:
+#      - uses: actions/checkout@v3
+#      - name: "Run tests with ghc8107: End-2-end test of hix project initialization and flakes development shell ..."
+#        run: "./test/tests.sh ghc8107 template"
+
   docs:
     runs-on: [self-hosted, linux]
     steps:
@@ -234,12 +243,11 @@ jobs:
       - name: "Check hix -- run github:haskell/cabal/3.10#cabal-install:exe:cabal -- --version"
         run: "HIX_DIR=$(mktemp -d) nix run .#hix --accept-flake-config -- run github:haskell/cabal/3.10#cabal-install:exe:cabal --accept-flake-config --override-input haskellNix . -- --version"
 
-  nix-tools-build:
+  nix-tools:
     runs-on: [self-hosted, linux]
     steps:
       - uses: actions/checkout@v3
-      - name: "nix-tools build"
-        run: cd ./nix-tools && .buildkite/nix-tools-build.sh
+      - run: nix build ./nix-tools#checks.x86_64-linux.truncate-index --accept-flake-config
 
   nixprof:
     runs-on: [self-hosted, linux]

.gitignore

@@ -78,7 +78,6 @@ cabal.sandbox.config
 *.hp
 *.eventlog
 .stack-work/
-cabal.project.local
 cabal.project.local~
 .HTF/
 .ghc.environment.*

build.nix

@@ -25,7 +25,7 @@ in rec {
 
   tools = pkgs.lib.optionalAttrs (ifdLevel >= 3) (
     pkgs.recurseIntoAttrs ({
-      cabal-latest = tool compiler-nix-name "cabal" { inherit evalPackages; };
+      cabal-latest = tool compiler-nix-name "cabal" { inherit evalPackages; cabalProjectLocal = builtins.readFile ./test/cabal.project.local; };
     } // pkgs.lib.optionalAttrs (__compareVersions haskell.compiler.${compiler-nix-name}.version "9.6" < 0) {
       hlint-latest = tool compiler-nix-name "hlint" {
         inherit evalPackages;
@@ -43,25 +43,29 @@ in rec {
             "ghc8107" = "3.4.1";
           }.${compiler-nix-name} or "latest";
       };
-    } // pkgs.lib.optionalAttrs (__compareVersions haskell.compiler.${compiler-nix-name}.version "9.6" < 0) {
+    } // pkgs.lib.optionalAttrs (
+             __compareVersions haskell.compiler.${compiler-nix-name}.version "9.2" >= 0
+          && __compareVersions haskell.compiler.${compiler-nix-name}.version "9.6" < 0) {
       stack =
         tool compiler-nix-name "stack" {
-          version =
-            if __compareVersions haskell.compiler.${compiler-nix-name}.version "9.2" < 0
-              then "2.9.3.1"
-              else "2.11.1";
+          version = "2.11.1";
           inherit evalPackages;
         };
     } // pkgs.lib.optionalAttrs (__compareVersions haskell.compiler.${compiler-nix-name}.version "9.6" < 0) {
       "hls-110" = tool compiler-nix-name "haskell-language-server" {
         inherit evalPackages;
         src = pkgs.haskell-nix.sources."hls-1.10";
       };
-    } // {
+    } // pkgs.lib.optionalAttrs (__compareVersions haskell.compiler.${compiler-nix-name}.version "9.8" < 0) {
       "hls-20" = tool compiler-nix-name "haskell-language-server" {
         inherit evalPackages;
         src = pkgs.haskell-nix.sources."hls-2.0";
       };
+    } // pkgs.lib.optionalAttrs (__compareVersions haskell.compiler.${compiler-nix-name}.version "9.8" < 0) {
+      "hls-22" = tool compiler-nix-name "haskell-language-server" {
+        inherit evalPackages;
+        src = pkgs.haskell-nix.sources."hls-2.2";
+      };
     })
   );
 
@@ -73,17 +77,15 @@ in rec {
     update-hackage = import ./scripts/update-hackage.nix {
       inherit (pkgs) stdenv lib writeScript coreutils glibc git
         openssh nixFlakes gawk bash curl findutils;
-      # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
-      nix-tools = haskell.internal-nix-tools;
-      cabal-install = haskell.internal-cabal-install;
+      # Update scripts use the internal nix-tools (compiled with a fixed GHC version)
+      nix-tools = haskell.nix-tools-unchecked;
       inherit (haskell) update-index-state-hashes cabal-issue-8352-workaround;
     };
     update-stackage = haskell.callPackage ./scripts/update-stackage.nix {
       inherit (pkgs) stdenv lib writeScript coreutils glibc git
         openssh nixFlakes gawk bash curl findutils;
-      # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
-      nix-tools = haskell.internal-nix-tools;
-      cabal-install = haskell.internal-cabal-install;
+      # Update scripts use the internal nix-tools (compiled with a fixed GHC version)
+      nix-tools = haskell.nix-tools-unchecked;
       inherit (haskell) cabal-issue-8352-workaround;
     };
     update-pins = haskell.callPackage ./scripts/update-pins.nix {};
@@ -94,7 +96,7 @@ in rec {
     check-closure-size = pkgs.buildPackages.callPackage ./scripts/check-closure-size.nix {
       # Includes cabal-install since this is commonly used.
       nix-tools = pkgs.linkFarm "common-tools" [
-        { name = "nix-tools";     path = haskell.nix-tools.${compiler-nix-name}; }
+        { name = "nix-tools";     path = haskell.nix-tools; }
         { name = "cabal-install"; path = haskell.cabal-install.${compiler-nix-name}; }
       ];
     };

builder/comp-builder.nix

@@ -39,6 +39,7 @@ let self =
 
 , dontPatchELF ? component.dontPatchELF
 , dontStrip ? component.dontStrip
+, dontUpdateAutotoolsGnuConfigScripts ? component.dontUpdateAutotoolsGnuConfigScripts
 , hardeningDisable ? component.hardeningDisable
 
 , enableStatic ? component.enableStatic
@@ -187,15 +188,23 @@ let
         "--with-ar=${stdenv.cc.bintools.targetPrefix}ar"
         "--with-strip=${stdenv.cc.bintools.targetPrefix}strip"
       ]
-    ) ++ [ # other flags
+    ) # Starting with ghc 9.10 the `ld command` will no longer be in the GHC `settings` file.
+      # We need to start passing it explicitly to setup like we do for `ar` and `strip`.
+      ++ lib.optional (!stdenv.hostPlatform.isGhcjs && builtins.compareVersions defaults.ghc.version "9.8" >= 0)
+        "--with-ld=${stdenv.cc.bintools.targetPrefix}ld"
+      ++ lib.optionals (stdenv.hostPlatform.isGhcjs) [
+        "--with-gcc=${buildPackages.emscripten}/bin/emcc"
+        "--with-ld=${buildPackages.emscripten}/bin/emcc"
+      ]
+      ++ [ # other flags
       (disableFeature dontStrip "executable-stripping")
       (disableFeature dontStrip "library-stripping")
       (enableFeature enableLibraryProfiling "library-profiling")
       (enableFeature enableProfiling "profiling")
       (enableFeature enableStatic "static")
       (enableFeature enableShared "shared")
       (enableFeature doCoverage "coverage")
-      (enableFeature enableLibraryForGhci "library-for-ghci")
+      (enableFeature (enableLibraryForGhci && !stdenv.hostPlatform.isGhcjs) "library-for-ghci")
     ] ++ lib.optionals (stdenv.hostPlatform.isMusl && (haskellLib.isExecutableType componentId)) [
       # These flags will make sure the resulting executable is statically linked.
       # If it uses other libraries it may be necessary for to add more
@@ -222,7 +231,7 @@ let
       ++ lib.optionals useLLVM [
         "--ghc-option=-fPIC" "--gcc-option=-fPIC"
         ]
-      ++ map (o: ''--ghc${lib.optionalString stdenv.hostPlatform.isGhcjs "js"}-options="${o}"'') ghcOptions
+      ++ map (o: ''--ghc${lib.optionalString (stdenv.hostPlatform.isGhcjs && builtins.compareVersions defaults.ghc.version "9" < 0) "js"}-options="${o}"'') ghcOptions
       ++ lib.optional (
         # GHC 9.2 cross compiler built with older versions of GHC seem to have problems
         # with unique conters.  Perhaps because the name changed for the counters.
@@ -260,8 +269,9 @@ let
                      if builtins.isFunction shellHook then shellHook { inherit package shellWrappers; }
                      else abort "shellHook should be a string or a function";
 
-  exeExt = if stdenv.hostPlatform.isGhcjs then ".jsexe/all.js" else
-    stdenv.hostPlatform.extensions.executable;
+  exeExt = if stdenv.hostPlatform.isGhcjs && builtins.compareVersions defaults.ghc.version "9.8" < 0
+    then ".jsexe/all.js"
+    else stdenv.hostPlatform.extensions.executable;
   exeName = componentId.cname + exeExt;
   testExecutable = "dist/build/${componentId.cname}/${exeName}";
 
@@ -294,10 +304,8 @@ let
         (if cabalFile != null
           then ''cp -v $cabalFilePath ${package.identifier.name}.cabal''
           else
-            # When building hpack package we use the internal nix-tools
-            # (compiled with a fixed GHC version)
             lib.optionalString (cabal-generator == "hpack") ''
-              ${buildPackages.haskell-nix.internal-nix-tools}/bin/hpack
+              ${buildPackages.haskell-nix.nix-tools-unchecked}/bin/hpack
             ''
         ) + lib.optionalString (prePatch != null) "\n${prePatch}";
     }
@@ -319,6 +327,9 @@ let
     // lib.optionalAttrs stdenv.hostPlatform.isMusl {
       # This fixes musl compilation of TH code that depends on C++ (for instance TH code that uses the double-conversion package)
       LD_LIBRARY_PATH="${pkgs.buildPackages.gcc-unwrapped.lib}/x86_64-unknown-linux-musl/lib";
+    }
+    // lib.optionalAttrs dontUpdateAutotoolsGnuConfigScripts {
+      inherit dontUpdateAutotoolsGnuConfigScripts;
     };
 
   haddock = haddockBuilder {
@@ -417,8 +428,12 @@ let
 
     prePatch =
       # emcc is very slow if it cannot cache stuff in $HOME
+      # Newer nixpkgs default the cache dir to nix store path.
+      # This seems to cause problems as it is not writeable.
+      # Setting EM_CACHE explicitly avoids this problem.
       (lib.optionalString stdenv.hostPlatform.isGhcjs ''
       export HOME=$(mktemp -d)
+      export EM_CACHE=$(mktemp -d)
       '') +
       (lib.optionalString (!canCleanSource) ''
       echo "Cleaning component source not supported, leaving it un-cleaned"
@@ -529,7 +544,7 @@ let
               # we assume that if the SETUP_HS command fails and the following line was found in the error
               # log, that it was the only error. Hence if we do _not_ find the line, grep will fail and this derivation
               # will be marked as failure.
-              cat $SETUP_ERR | grep 'Error: Setup: No executables and no library found\. Nothing to do\.'
+              cat $SETUP_ERR | grep 'No executables and no library found\. Nothing to do\.'
             fi
             ''}
       ${lib.optionalString (haskellLib.isLibrary componentId) ''
@@ -581,13 +596,13 @@ let
               '')
         }
       ''}
-      ${(lib.optionalString (haskellLib.isTest componentId || haskellLib.isBenchmark componentId) ''
+      ${(lib.optionalString (haskellLib.isTest componentId || haskellLib.isBenchmark componentId || (haskellLib.isExe componentId && stdenv.hostPlatform.isGhcjs)) ''
         mkdir -p $out/bin
         if [ -f ${testExecutable} ]; then
           mkdir -p $(dirname $out/bin/${exeName})
           ${lib.optionalString stdenv.buildPlatform.isLinux "sync"}
-          ${if stdenv.hostPlatform.isGhcjs then ''
-            cat <(echo \#!${lib.getBin buildPackages.nodejs-18_x}/bin/node) ${testExecutable} >| $out/bin/${exeName}
+          ${if stdenv.hostPlatform.isGhcjs && builtins.compareVersions defaults.ghc.version "9.8" < 0 then ''
+            cat <(echo \#!/usr/bin/env node) ${testExecutable} >| $out/bin/${exeName}
             chmod +x $out/bin/${exeName}
           '' else ''
              cp -r ${testExecutable} $(dirname $out/bin/${exeName})
@@ -603,7 +618,10 @@ let
         for p in ${lib.concatStringsSep " " ([ libffi gmp ] ++
               # Also include C++ and mcfgthreads DLLs for GHC 9.4.1 and newer
               lib.optionals (builtins.compareVersions defaults.ghc.version "9.4.1" >= 0)
-                [ buildPackages.gcc-unwrapped windows.mcfgthreads ])}; do
+                [ buildPackages.gcc-unwrapped
+                  # Find the versions of mcfgthreads used by stdenv.cc
+                  (pkgs.threadsCrossFor or (x: { package = pkgs.windows.mcfgthreads; }) pkgs.stdenv.cc.version).package
+                ])}; do
           find "$p" -iname '*.dll' -exec ln -s {} $out/bin \;
         done
         ''

builder/default.nix

@@ -64,10 +64,11 @@ let
       then pkgs.path
       else pkgs.haskell-nix.sources.nixpkgs-2205;
     nixpkgsHoogle = import (nixpkgs + /pkgs/development/haskell-modules/hoogle.nix);
-  in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.tool "ghc8107" "hoogle" {
+  in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.tool "ghc928" "hoogle" {
         inherit evalPackages;
         version = "5.0.18.3";
-        index-state = pkgs.haskell-nix.internalHackageIndexState;
+        # index-state = pkgs.haskell-nix.internalHackageIndexState;
+        index-state = "2023-06-05T00:00:00Z";
       }
     }:
     let

builder/make-config-files.nix

@@ -48,7 +48,11 @@ let
   ghcCommand'    = if isGhcjs then "ghcjs" else "ghc";
   ghcCommand     = "${ghc.targetPrefix}${ghcCommand'}";
   ghcCommandCaps = lib.toUpper ghcCommand';
-  libDir         = ghc.libDir or "lib/${ghcCommand}-${ghc.version}";
+  libDir         = ghc.libDir or
+    # nixpkgs versions of `ghc` do not have a `.libDir`.  So this
+    # default is for them.
+    ("lib/${ghcCommand}-${ghc.version}"
+      + lib.optionalString (__compareVersions ghc.version "9.6.1" >= 0) "/lib");
   packageCfgDir  = "${libDir}/package.conf.d";
 
   libDeps = haskellLib.uniqueWithName (
@@ -81,9 +85,14 @@ let
         find $unwrappedGhc/${packageCfgDir} -name $p'*.conf' -exec cp -f {} $configFiles/${packageCfgDir} \;
       done
     ''}
+    ${ # From GHC 9.6 the nixpkgs ghc derviations now use ${pkgroot} in their `.conf` files.
+    ''
+      sed -i 's|''${pkgroot}/../../../../|/nix/store/|' $configFiles/${packageCfgDir}/*.conf
+      sed -i 's|''${pkgroot}|${ghc}/${packageCfgDir}/..|' $configFiles/${packageCfgDir}/*.conf
+    ''}
 
     for l in "''${pkgsHostTarget[@]}"; do
-      if [ -d "$l/${packageCfgDir}" ]; then
+      if [ -d "$l/${packageCfgDir}" ] && [[ "$l" != "${ghc}" ]]; then
         files=("$l/${packageCfgDir}/"*.conf)
         if (( ''${#files[@]} )); then
           cp -f "''${files[@]}" $configFiles/${packageCfgDir}

ci-lib.nix

@@ -1,8 +1,5 @@
-{
-  # Generic nixpkgs, use *only* for lib functions that are stable across versions
-  pkgs,
-  lib ? pkgs.lib
-}: rec {
+{ lib }:
+rec {
   inherit (import ./dimension.nix) dimension;
 
   # A filter for removing packages that aren't supported on the current platform