From ba4e8c7ad57bd0830c294602c8ba4dac8aeac8cd Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Sat, 6 Apr 2019 06:44:02 +0200 Subject: [PATCH] Faster, more succeeding builds, better everything - Now fetching nixpgks with fetchFromGitHub by computing and saving the sha256 at update time - Detecting core GHC libraries and setting them to null to fix builds, workaround for https://github.com/input-output-hk/stack2nix/issues/134 - Add override directory to fix builds for specific ghc versions - Speed up builds by disabling library profiling for everything - New nix file structure with powerful selection mechanisms --- .gitignore | 1 + Design.org | 26 +++++++++---- default.nix | 110 ++++++++++++++++++++++++++++++++++++++++++---------- update.hs | 60 ++++++++++++++++++++++++---- 4 files changed, 162 insertions(+), 35 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fcfc4a1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +result* diff --git a/Design.org b/Design.org index 6c518d8..533d1b4 100644 --- a/Design.org +++ b/Design.org @@ -5,21 +5,31 @@ Assumption: The stack build does not depend on which nixpkgs version it uses, it - history-nixpkgs-unstable - nixpkgs - haskell-ide-engine +- per-ghcMinor + - : File with the list of base libraries for that ghc version - per-nixpkgs - - : File with a list of ghc versions this nixpkgs rev has + - ghcVersions + - : File with a list of ghc versions this nixpkgs rev has + - sha256 + - : sha256 hash for fetchFromGitHub - per-haskell-ide-engine - - .nix: File generated by stack2nix for hie-rev and ghcVersion -./generated/ layout: +./ layout: - nixpkgsForGhc - : File containing the nixpkgsrev to use for that ghc -- stable - - .nix: File copied from the cache - - revision: File with the git revision of current stable hie -- unstable - - .nix: File copied from the cache - - revision: File with the git revision of current unstable hie +- nixpkgsHashes + - : File containing the sha256 hash for fetchFromGitHub for a nixpkgs version +- ghcBaseLibraries: + - : List of base libraries for that ghc version +- versions + - stable + - .nix: File copied from the cache + - revision: File with the git revision of current stable hie + - unstable + - .nix: File copied from the cache + - revision: File with the git revision of current unstable hie Flags: - Whether master should be regenerated diff --git a/default.nix b/default.nix index ac62929..6a9afbd 100644 --- a/default.nix +++ b/default.nix @@ -3,36 +3,106 @@ with builtins; let - nixpkgsForGhc = mapAttrs (file: _: readFile (./nixpkgsForGhc + "/${file}")) + + pkgsForGhc = mapAttrs (ghc: _: let + rev = readFile (./nixpkgsForGhc + "/${ghc}"); + sha256 = readFile (./nixpkgsHashes + "/${rev}"); + in + import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz"; + inherit sha256; + }) { + config = {}; + overlays = []; + }) (readDir ./nixpkgsForGhc); version = name: mapAttrs' (file: _: let ghcVersion = removeSuffix ".nix" file; - pkgs = import (fetchGit { - url = "https://github.com/NixOS/nixpkgs"; - rev = nixpkgsForGhc.${ghcVersion}; - }) {}; - build = pkgs.haskell.lib.justStaticExecutables - (import (./versions + "/${name}/${file}") { + pkgs = pkgsForGhc.${ghcVersion}; + inherit (pkgs) lib haskell; + + overrideFun = old: { + overrides = lib.composeExtensions + (lib.composeExtensions + (old.overrides or (self: super: {})) + (hself: hsuper: { + + # Disable library profiling for faster builds + mkDerivation = args: hsuper.mkDerivation (args // { + enableLibraryProfiling = false; + }); + + # Embed the ghc version into the name + haskell-ide-engine = haskell.lib.overrideCabal hsuper.haskell-ide-engine (old: { + pname = "${old.pname}-${ghcVersion}"; + }); + } + // lib.flip genAttrs (name: null) + (lib.filter (name: name != "ghc" && name != "Cabal") + (map (name: (builtins.parseDrvName name).name) + (lib.splitString " " + (builtins.readFile (./ghcBaseLibraries + "/${ghcVersion}"))))) + ) + ) + (if builtins.pathExists (./overrides + "/${ghcVersion}.nix") then + import (./overrides + "/${ghcVersion}.nix") + else self: super: {} + ); + }; + + build = haskell.lib.justStaticExecutables + ((import (./versions + "/${name}/${file}") { inherit pkgs; - }).haskell-ide-engine; - in if hasSuffix ".nix" file then { + }).override overrideFun).haskell-ide-engine; + in { name = ghcVersion; value = build; - } else { - name = file; - value = builtins.readFile (./versions + "/${name}/${file}"); - }) - (readDir (./versions + "/${name}")); - -in rec { + }) + (filterAttrs (file: _: file != "revision") (readDir (./versions + "/${name}"))); versions = mapAttrs (file: _: version file) (readDir ./versions); - forGhc = versions.unstable // versions.stable; + pkgs = import {}; + lib = pkgs.lib; + + parseNixGhcVersion = version: + lib.concatStringsSep "." (builtins.match "ghc(.)(.)(.)" version); + + combine = versions: let + wrapperVersion = lib.last (lib.attrNames versions); + in pkgs.runCommand "haskell-ide-engine-combined" { + nativeBuildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + + ${concatMapStringsSep "\n" (ghcVersion: '' + ln -s ${versions.${ghcVersion}}/bin/hie $out/bin/hie-${parseNixGhcVersion ghcVersion} + '') (lib.attrNames versions)} + + makeWrapper ${versions.${wrapperVersion}}/bin/hie-wrapper $out/bin/hie-wrapper \ + --suffix PATH : $out/bin + ln -s hie-wrapper $out/bin/hie + ''; + + inherit (versions) stable unstable; + + makeSet = versions: combine versions // { + inherit versions; + select = selector: makeSet (selector versions); + minors = mapAttrs (name: makeSet) + (foldl' (acc: el: let minor = lib.substring 0 (lib.stringLength el - 1) el; in + acc // { + ${minor} = acc.${minor} or {} // { ${el} = versions.${el}; }; + } + ) {} (lib.attrNames versions)); + }; - # forGhcs (p: [ p.ghc864 p.ghc863 ]) - forGhcs = selector: null; + result = makeSet (unstable // stable) // { + onlyStable = makeSet stable; + onlyUnstable = makeSet unstable; + allVersions = versions; + }; -} +in result diff --git a/update.hs b/update.hs index 68f9e3a..532e859 100755 --- a/update.hs +++ b/update.hs @@ -28,10 +28,13 @@ import System.Process import Text.Regex.Applicative data Env = Env - { gitBin :: FilePath - , nixBin :: FilePath - , cacheDir :: FilePath - , manager :: Manager + { gitBin :: FilePath + , nixBin :: FilePath + , nixHashBin :: FilePath + , nixBuildBin :: FilePath + , cpBin :: FilePath + , cacheDir :: FilePath + , manager :: Manager } -- | Initializes the read-only environment @@ -39,6 +42,9 @@ getEnv :: IO Env getEnv = Env <$> (fromMaybe (error "Can't find git executable") <$> findExecutable "git") <*> (fromMaybe (error "Can't find nix-instantiate executable") <$> findExecutable "nix-instantiate") + <*> (fromMaybe (error "Can't find nix-hash executable") <$> findExecutable "nix-hash") + <*> (fromMaybe (error "Can't find nix-build executable") <$> findExecutable "nix-build") + <*> (fromMaybe (error "Can't find cp executable") <$> findExecutable "cp") <*> getXdgDirectory XdgCache "all-hies" <*> newTlsManager @@ -64,9 +70,13 @@ run = do setupDirectories :: App () setupDirectories = do - cachePath "per-nixpkgs" >>= liftIO . createDirectoryIfMissing True + cachePath "per-nixpkgs/ghcVersions" >>= liftIO . createDirectoryIfMissing True + cachePath "per-nixpkgs/sha256" >>= liftIO . createDirectoryIfMissing True cachePath "per-hie" >>= liftIO . createDirectoryIfMissing True + cachePath "per-ghcMinor" >>= liftIO . createDirectoryIfMissing True liftIO $ createDirectoryIfMissing True "nixpkgsForGhc" + cleanDirectory "ghcBaseLibraries" + cleanDirectory "nixpkgsHashes" -- | Makes sure that the given directory is existent and empty cleanDirectory :: FilePath -> App () @@ -122,6 +132,10 @@ genS2N hash version = do exists <- liftIO $ doesFileExist path nixpkgsRev <- findNixpkgsForGhc version + sha <- nixpkgsSha nixpkgsRev + liftIO $ writeFile ("nixpkgsHashes" nixpkgsRev) sha + + genBaseLibraries version nixpkgsRev unless exists $ do liftIO $ putStrLn $ "Using nixpkgs revision " ++ nixpkgsRev ++ " for ghc version " ++ show version @@ -129,6 +143,38 @@ genS2N hash version = do callStack2nix hash path version return path +genBaseLibraries :: Version -> String -> App () +genBaseLibraries version@(Version major minor patch) nixpkgsRev = do + cache <- cachePath $ "per-ghcMinor" show major ++ show minor + exists <- liftIO $ doesFileExist cache + unless exists $ do + git nixpkgs [ "checkout", nixpkgsRev ] + nix <- lift $ asks nixBuildBin + ghcPath <- liftIO $ init <$> readProcess nix + [ "--no-out-link", "", "-A", "haskell.compiler." ++ nixVersion version ] "" + libs <- liftIO $ readProcess (ghcPath "bin/ghc-pkg") + [ "list", "--no-user-package-db", "--simple" ] "" + liftIO $ writeFile cache libs + liftIO $ copyFile cache ("ghcBaseLibraries" nixVersion version) + +nixpkgsSha :: String -> App String +nixpkgsSha revision = do + cacheFile <- cachePath $ "per-nixpkgs/sha256" revision + exists <- liftIO $ doesFileExist cacheFile + if exists then liftIO $ readFile cacheFile + else do + git nixpkgs [ "checkout", revision ] + path <- repoPath nixpkgs + tmp <- cachePath "tmp" + cp <- lift $ asks cpBin + liftIO $ readProcess cp ["-rl", path, tmp] "" + liftIO $ removeDirectoryRecursive (tmp ".git") + nixHash <- lift $ asks nixHashBin + hash <- liftIO $ head . lines <$> readProcess nixHash ["--type", "sha256", "--base32", tmp] "" + liftIO $ writeFile cacheFile hash + liftIO $ removeDirectoryRecursive tmp + return hash + -- | Finds a suitable nixpkgs revision that has a the specified compiler available findNixpkgsForGhc :: Version -> App String findNixpkgsForGhc version = do @@ -154,7 +200,7 @@ findNixpkgsForGhc version = do -- | Determines the available GHC versions for a nixpkgs revision ghcVersionsForNixpkgs :: String -> App [Version] ghcVersionsForNixpkgs rev = do - path <- cachePath $ "per-nixpkgs" rev + path <- cachePath $ "per-nixpkgs/ghcVersions" rev exists <- liftIO $ doesFileExist path if exists then do contents <- liftIO $ readFile path @@ -253,7 +299,7 @@ getHistory = do response <- liftIO $ responseBody <$> httpLbs (parseRequest_ url) mgr -- Because this file is downloaded in the order of oldest to newest -- We reverse it for easier processing - let items = reverse . map (head . BS.words) $ BS.lines response + let items = reverse . ("45e819dd49799d8b680084ed57f6d0633581b957":) . map (head . BS.words) $ BS.lines response liftIO $ BS.writeFile path $ BS.unlines items return $ map BS.unpack items