Skip to content
This repository has been archived by the owner on Feb 13, 2021. It is now read-only.

Commit

Permalink
Faster, more succeeding builds, better everything
Browse files Browse the repository at this point in the history
- Now fetching nixpgks with fetchFromGitHub by computing and saving the
sha256 at update time
- Detecting core GHC libraries and setting them to null to fix builds,
workaround for input-output-hk/stack2nix#134
- Add override directory to fix builds for specific ghc versions
- Speed up builds by disabling library profiling for everything
- New nix file structure with powerful selection mechanisms
  • Loading branch information
infinisil committed Apr 6, 2019
1 parent 199a76b commit ba4e8c7
Show file tree
Hide file tree
Showing 4 changed files with 162 additions and 35 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
result*
26 changes: 18 additions & 8 deletions Design.org
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,31 @@ Assumption: The stack build does not depend on which nixpkgs version it uses, it
- history-nixpkgs-unstable
- nixpkgs
- haskell-ide-engine
- per-ghcMinor
- <major><minor>: File with the list of base libraries for that ghc version
- per-nixpkgs
- <nixpkgsrev-ghcVersions>: File with a list of ghc versions this nixpkgs rev has
- ghcVersions
- <nixpkgsrev>: File with a list of ghc versions this nixpkgs rev has
- sha256
- <nixpkgsRev>: sha256 hash for fetchFromGitHub
- per-haskell-ide-engine
- <hie-rev>
- <ghcVersion-stack2nix>.nix: File generated by stack2nix for hie-rev and ghcVersion

./generated/ layout:
./ layout:
- nixpkgsForGhc
- <ghcVersion>: File containing the nixpkgsrev to use for that ghc
- stable
- <ghcVersion>.nix: File copied from the cache
- revision: File with the git revision of current stable hie
- unstable
- <ghcVersion>.nix: File copied from the cache
- revision: File with the git revision of current unstable hie
- nixpkgsHashes
- <nixpkgsRevision>: File containing the sha256 hash for fetchFromGitHub for a nixpkgs version
- ghcBaseLibraries:
- <ghcVersion>: List of base libraries for that ghc version
- versions
- stable
- <ghcVersion>.nix: File copied from the cache
- revision: File with the git revision of current stable hie
- unstable
- <ghcVersion>.nix: File copied from the cache
- revision: File with the git revision of current unstable hie

Flags:
- Whether master should be regenerated
Expand Down
110 changes: 90 additions & 20 deletions default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -3,36 +3,106 @@ with builtins;

let

nixpkgsForGhc = mapAttrs (file: _: readFile (./nixpkgsForGhc + "/${file}"))

pkgsForGhc = mapAttrs (ghc: _: let
rev = readFile (./nixpkgsForGhc + "/${ghc}");
sha256 = readFile (./nixpkgsHashes + "/${rev}");
in
import (fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
inherit sha256;
}) {
config = {};
overlays = [];
})
(readDir ./nixpkgsForGhc);

version = name:
mapAttrs' (file: _: let
ghcVersion = removeSuffix ".nix" file;
pkgs = import (fetchGit {
url = "https://github.com/NixOS/nixpkgs";
rev = nixpkgsForGhc.${ghcVersion};
}) {};
build = pkgs.haskell.lib.justStaticExecutables
(import (./versions + "/${name}/${file}") {
pkgs = pkgsForGhc.${ghcVersion};
inherit (pkgs) lib haskell;

overrideFun = old: {
overrides = lib.composeExtensions
(lib.composeExtensions
(old.overrides or (self: super: {}))
(hself: hsuper: {

# Disable library profiling for faster builds
mkDerivation = args: hsuper.mkDerivation (args // {
enableLibraryProfiling = false;
});

# Embed the ghc version into the name
haskell-ide-engine = haskell.lib.overrideCabal hsuper.haskell-ide-engine (old: {
pname = "${old.pname}-${ghcVersion}";
});
}
// lib.flip genAttrs (name: null)
(lib.filter (name: name != "ghc" && name != "Cabal")
(map (name: (builtins.parseDrvName name).name)
(lib.splitString " "
(builtins.readFile (./ghcBaseLibraries + "/${ghcVersion}")))))
)
)
(if builtins.pathExists (./overrides + "/${ghcVersion}.nix") then
import (./overrides + "/${ghcVersion}.nix")
else self: super: {}
);
};

build = haskell.lib.justStaticExecutables
((import (./versions + "/${name}/${file}") {
inherit pkgs;
}).haskell-ide-engine;
in if hasSuffix ".nix" file then {
}).override overrideFun).haskell-ide-engine;
in {
name = ghcVersion;
value = build;
} else {
name = file;
value = builtins.readFile (./versions + "/${name}/${file}");
})
(readDir (./versions + "/${name}"));

in rec {
})
(filterAttrs (file: _: file != "revision") (readDir (./versions + "/${name}")));

versions = mapAttrs (file: _: version file) (readDir ./versions);

forGhc = versions.unstable // versions.stable;
pkgs = import <nixpkgs> {};
lib = pkgs.lib;

parseNixGhcVersion = version:
lib.concatStringsSep "." (builtins.match "ghc(.)(.)(.)" version);

combine = versions: let
wrapperVersion = lib.last (lib.attrNames versions);
in pkgs.runCommand "haskell-ide-engine-combined" {
nativeBuildInputs = [ pkgs.makeWrapper ];
} ''
mkdir -p $out/bin
${concatMapStringsSep "\n" (ghcVersion: ''
ln -s ${versions.${ghcVersion}}/bin/hie $out/bin/hie-${parseNixGhcVersion ghcVersion}
'') (lib.attrNames versions)}
makeWrapper ${versions.${wrapperVersion}}/bin/hie-wrapper $out/bin/hie-wrapper \
--suffix PATH : $out/bin
ln -s hie-wrapper $out/bin/hie
'';

inherit (versions) stable unstable;

makeSet = versions: combine versions // {
inherit versions;
select = selector: makeSet (selector versions);
minors = mapAttrs (name: makeSet)
(foldl' (acc: el: let minor = lib.substring 0 (lib.stringLength el - 1) el; in
acc // {
${minor} = acc.${minor} or {} // { ${el} = versions.${el}; };
}
) {} (lib.attrNames versions));
};

# forGhcs (p: [ p.ghc864 p.ghc863 ])
forGhcs = selector: null;
result = makeSet (unstable // stable) // {
onlyStable = makeSet stable;
onlyUnstable = makeSet unstable;
allVersions = versions;
};

}
in result
60 changes: 53 additions & 7 deletions update.hs
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,23 @@ import System.Process
import Text.Regex.Applicative

data Env = Env
{ gitBin :: FilePath
, nixBin :: FilePath
, cacheDir :: FilePath
, manager :: Manager
{ gitBin :: FilePath
, nixBin :: FilePath
, nixHashBin :: FilePath
, nixBuildBin :: FilePath
, cpBin :: FilePath
, cacheDir :: FilePath
, manager :: Manager
}

-- | Initializes the read-only environment
getEnv :: IO Env
getEnv = Env
<$> (fromMaybe (error "Can't find git executable") <$> findExecutable "git")
<*> (fromMaybe (error "Can't find nix-instantiate executable") <$> findExecutable "nix-instantiate")
<*> (fromMaybe (error "Can't find nix-hash executable") <$> findExecutable "nix-hash")
<*> (fromMaybe (error "Can't find nix-build executable") <$> findExecutable "nix-build")
<*> (fromMaybe (error "Can't find cp executable") <$> findExecutable "cp")
<*> getXdgDirectory XdgCache "all-hies"
<*> newTlsManager

Expand All @@ -64,9 +70,13 @@ run = do

setupDirectories :: App ()
setupDirectories = do
cachePath "per-nixpkgs" >>= liftIO . createDirectoryIfMissing True
cachePath "per-nixpkgs/ghcVersions" >>= liftIO . createDirectoryIfMissing True
cachePath "per-nixpkgs/sha256" >>= liftIO . createDirectoryIfMissing True
cachePath "per-hie" >>= liftIO . createDirectoryIfMissing True
cachePath "per-ghcMinor" >>= liftIO . createDirectoryIfMissing True
liftIO $ createDirectoryIfMissing True "nixpkgsForGhc"
cleanDirectory "ghcBaseLibraries"
cleanDirectory "nixpkgsHashes"

-- | Makes sure that the given directory is existent and empty
cleanDirectory :: FilePath -> App ()
Expand Down Expand Up @@ -122,13 +132,49 @@ genS2N hash version = do
exists <- liftIO $ doesFileExist path

nixpkgsRev <- findNixpkgsForGhc version
sha <- nixpkgsSha nixpkgsRev
liftIO $ writeFile ("nixpkgsHashes" </> nixpkgsRev) sha

genBaseLibraries version nixpkgsRev

unless exists $ do
liftIO $ putStrLn $ "Using nixpkgs revision " ++ nixpkgsRev ++ " for ghc version " ++ show version
git nixpkgs [ "checkout", nixpkgsRev ]
callStack2nix hash path version
return path

genBaseLibraries :: Version -> String -> App ()
genBaseLibraries version@(Version major minor patch) nixpkgsRev = do
cache <- cachePath $ "per-ghcMinor" </> show major ++ show minor
exists <- liftIO $ doesFileExist cache
unless exists $ do
git nixpkgs [ "checkout", nixpkgsRev ]
nix <- lift $ asks nixBuildBin
ghcPath <- liftIO $ init <$> readProcess nix
[ "--no-out-link", "<nixpkgs>", "-A", "haskell.compiler." ++ nixVersion version ] ""
libs <- liftIO $ readProcess (ghcPath </> "bin/ghc-pkg")
[ "list", "--no-user-package-db", "--simple" ] ""
liftIO $ writeFile cache libs
liftIO $ copyFile cache ("ghcBaseLibraries" </> nixVersion version)

nixpkgsSha :: String -> App String
nixpkgsSha revision = do
cacheFile <- cachePath $ "per-nixpkgs/sha256" </> revision
exists <- liftIO $ doesFileExist cacheFile
if exists then liftIO $ readFile cacheFile
else do
git nixpkgs [ "checkout", revision ]
path <- repoPath nixpkgs
tmp <- cachePath "tmp"
cp <- lift $ asks cpBin
liftIO $ readProcess cp ["-rl", path, tmp] ""
liftIO $ removeDirectoryRecursive (tmp </> ".git")
nixHash <- lift $ asks nixHashBin
hash <- liftIO $ head . lines <$> readProcess nixHash ["--type", "sha256", "--base32", tmp] ""
liftIO $ writeFile cacheFile hash
liftIO $ removeDirectoryRecursive tmp
return hash

-- | Finds a suitable nixpkgs revision that has a the specified compiler available
findNixpkgsForGhc :: Version -> App String
findNixpkgsForGhc version = do
Expand All @@ -154,7 +200,7 @@ findNixpkgsForGhc version = do
-- | Determines the available GHC versions for a nixpkgs revision
ghcVersionsForNixpkgs :: String -> App [Version]
ghcVersionsForNixpkgs rev = do
path <- cachePath $ "per-nixpkgs" </> rev
path <- cachePath $ "per-nixpkgs/ghcVersions" </> rev
exists <- liftIO $ doesFileExist path
if exists then do
contents <- liftIO $ readFile path
Expand Down Expand Up @@ -253,7 +299,7 @@ getHistory = do
response <- liftIO $ responseBody <$> httpLbs (parseRequest_ url) mgr
-- Because this file is downloaded in the order of oldest to newest
-- We reverse it for easier processing
let items = reverse . map (head . BS.words) $ BS.lines response
let items = reverse . ("45e819dd49799d8b680084ed57f6d0633581b957":) . map (head . BS.words) $ BS.lines response

liftIO $ BS.writeFile path $ BS.unlines items
return $ map BS.unpack items
Expand Down

0 comments on commit ba4e8c7

Please sign in to comment.