Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev environments #12

Merged
merged 9 commits into from
Feb 1, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,4 @@ before_script:
script:
- nix-env -if https://github.com/cachix/cachix/tarball/master --extra-substituters https://cachix.cachix.org --trusted-public-keys 'cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cachix.cachix.org-1:eWNHQldwUO7G2VkjpnjDbWwy4KQ/HNxht7H4SSoMckM='
- cachix use nix-tools
- nix-build
-I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/${CHANNEL}.tar.gz
test/default.nix
--cores 0
-j2
- NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/${CHANNEL}.tar.gz NIX_BUILD_ARGS="--cores 0 --max-jobs 2" ./test/tests.sh
70 changes: 54 additions & 16 deletions builder/comp-builder.nix
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{ stdenv, buildPackages, ghc, lib, pkgconfig, writeText, runCommand, haskellLib, nonReinstallablePkgs }:
{ stdenv, buildPackages, ghc, lib, pkgconfig, writeText, runCommand, haskellLib, nonReinstallablePkgs, withPackage }:

{ componentId
, component
Expand All @@ -16,8 +16,9 @@
, preBuild ? null, postBuild ? null
, preCheck ? null, postCheck ? null
, preInstall ? null, postInstall ? null
, shellHook ? null

, doCheck ? component.doCheck || componentId.ctype == "test"
, doCheck ? component.doCheck || haskellLib.isTest componentId
, doCrossCheck ? component.doCrossCheck || false
, dontPatchELF ? true
, dontStrip ? true
Expand All @@ -27,7 +28,9 @@
}:

let
fullName = "${name}-${componentId.ctype}-${componentId.cname}";
fullName = if haskellLib.isAll componentId
then "${name}-all"
else "${name}-${componentId.ctype}-${componentId.cname}";

flagsAndConfig = field: xs: lib.optionalString (xs != []) ''
echo ${lib.concatStringsSep " " (map (x: "--${field}=${x}") xs)} >> $out/configure-flags
Expand All @@ -44,18 +47,30 @@ let
in map ({val,...}: val) closure;

exactDep = pdbArg: p: ''
if id=$(${ghc.targetPrefix}ghc-pkg -v0 ${pdbArg} field ${p} id --simple-output); then
if id=$(target-pkg ${pdbArg} field ${p} id --simple-output); then
echo "--dependency=${p}=$id" >> $out/configure-flags
fi
if ver=$(${ghc.targetPrefix}ghc-pkg -v0 ${pdbArg} field ${p} version --simple-output); then
if ver=$(target-pkg ${pdbArg} field ${p} version --simple-output); then
echo "constraint: ${p} == $ver" >> $out/cabal.config
echo "constraint: ${p} installed" >> $out/cabal.config
fi
'';

envDep = pdbArg: p: ''
if id=$(target-pkg ${pdbArg} field ${p} id --simple-output); then
echo "package-id $id" >> $out/ghc-environment
fi
'';

configFiles = runCommand "${fullName}-config" { nativeBuildInputs = [ghc]; } (''
mkdir -p $out
${ghc.targetPrefix}ghc-pkg -v0 init $out/package.conf.d

# Calls ghc-pkg for the target platform
target-pkg() {
${ghc.targetPrefix}ghc-pkg "$@"
}

target-pkg init $out/package.conf.d

${lib.concatStringsSep "\n" (lib.mapAttrsToList flagsAndConfig {
"extra-lib-dirs" = map (p: "${lib.getLib p}/lib") component.libs;
Expand All @@ -67,18 +82,26 @@ let
# Note: we need to use --global-package-db with ghc-pkg to prevent it
# from looking into the implicit global package db when registering the package.
${lib.concatMapStringsSep "\n" (p: ''
${ghc.targetPrefix}ghc-pkg -v0 describe ${p} | ${ghc.targetPrefix}ghc-pkg -v0 --force --global-package-db $out/package.conf.d register - || true
target-pkg describe ${p} | target-pkg --force --global-package-db $out/package.conf.d register - || true
'') nonReinstallablePkgs}

${lib.concatMapStringsSep "\n" (p: ''
${ghc.targetPrefix}ghc-pkg -v0 --package-db ${p}/package.conf.d dump | ${ghc.targetPrefix}ghc-pkg -v0 --force --package-db $out/package.conf.d register -
target-pkg --package-db ${p}/package.conf.d dump | target-pkg --force --package-db $out/package.conf.d register -
'') flatDepends}

# Note: we pass `clear` first to ensure that we never consult the implicit global package db.
${flagsAndConfig "package-db" ["clear" "$out/package.conf.d"]}

echo ${lib.concatStringsSep " " (lib.mapAttrsToList (fname: val: "--flags=${lib.optionalString (!val) "-" + fname}") flags)} >> $out/configure-flags

# Provide a GHC environment file
cat > $out/ghc-environment <<EOF
clear-package-db
package-db $out/package.conf.d
EOF
${lib.concatMapStringsSep "\n" (p: envDep "--package-db ${p.components.library}/package.conf.d" p.identifier.name) component.depends}
${lib.concatMapStringsSep "\n" (envDep "") (lib.remove "ghc" nonReinstallablePkgs)}

'' + lib.optionalString component.doExactConfig ''
echo "--exact-configuration" >> $out/configure-flags
echo "allow-newer: ${package.identifier.name}:*" >> $out/cabal.config
Expand Down Expand Up @@ -115,14 +138,14 @@ let
sed -i "s,dynamic-library-dirs: .*,dynamic-library-dirs: $dynamicLinksDir," $f
done
'' + ''
${ghc.targetPrefix}ghc-pkg -v0 --package-db $out/package.conf.d recache
target-pkg --package-db $out/package.conf.d recache
'' + ''
${ghc.targetPrefix}ghc-pkg -v0 --package-db $out/package.conf.d check
target-pkg --package-db $out/package.conf.d check
'');

finalConfigureFlags = lib.concatStringsSep " " (
[ "--prefix=$out"
"${componentId.ctype}:${componentId.cname}"
"${haskellLib.componentTarget componentId}"
"$(cat ${configFiles}/configure-flags)"
# GHC
"--with-ghc=${ghc.targetPrefix}ghc"
Expand All @@ -145,6 +168,16 @@ let
++ component.configureFlags
);

executableToolDepends = lib.concatMap (c: if c.isHaskell or false
then builtins.attrValues (c.components.exes or {})
else [c]) component.build-tools;

# Unfortunately, we need to wrap ghc commands for cabal builds to
# work in the nix-shell. See ../doc/removing-with-package-wrapper.md.
shellWrappers = withPackage {
inherit package configFiles;
};

in stdenv.mkDerivation ({
name = fullName;

Expand All @@ -154,6 +187,7 @@ in stdenv.mkDerivation ({
inherit (package) identifier;
config = component;
inherit configFiles;
env = shellWrappers;
};

meta = {
Expand All @@ -167,6 +201,7 @@ in stdenv.mkDerivation ({
};

CABAL_CONFIG = configFiles + /cabal.config;
GHC_ENVIRONMENT = configFiles + /ghc-environment;
LANG = "en_US.UTF-8"; # GHC needs the locale configured during the Haddock phase.
LC_ALL = "en_US.UTF-8";

Expand All @@ -179,9 +214,7 @@ in stdenv.mkDerivation ({
nativeBuildInputs =
[ghc]
++ lib.optional (component.pkgconfig != []) pkgconfig
++ lib.concatMap (c: if c.isHaskell or false
then builtins.attrValues (c.components.exes or {})
else [c]) component.build-tools;
++ executableToolDepends;

SETUP_HS = setup + /bin/Setup;

Expand Down Expand Up @@ -217,12 +250,12 @@ in stdenv.mkDerivation ({
installPhase = ''
runHook preInstall
$SETUP_HS copy ${lib.concatStringsSep " " component.setupInstallFlags}
${lib.optionalString (haskellLib.isLibrary componentId) ''
${lib.optionalString (haskellLib.isLibrary componentId || haskellLib.isAll componentId) ''
$SETUP_HS register --gen-pkg-config=${name}.conf
${ghc.targetPrefix}ghc-pkg -v0 init $out/package.conf.d
${ghc.targetPrefix}ghc-pkg -v0 --package-db ${configFiles}/package.conf.d -f $out/package.conf.d register ${name}.conf
''}
${lib.optionalString (componentId.ctype == "test") ''
${lib.optionalString (haskellLib.isTest componentId || haskellLib.isAll componentId) ''
mkdir -p $out/${name}
if [ -f "dist/build/${componentId.cname}/${componentId.cname}" ]; then
cp dist/build/${componentId.cname}/${componentId.cname} $out/${name}/
Expand All @@ -233,6 +266,11 @@ in stdenv.mkDerivation ({
''}
runHook postInstall
'';

shellHook = ''
export PATH="${shellWrappers}/bin:$PATH"
${toString shellHook}
'';
}
# patches can (if they like) depend on the version and revision of the package.
// lib.optionalAttrs (patches != []) { patches = map (p: if builtins.isFunction p then p { inherit (package.identifier) version; inherit revision; } else p) patches; }
Expand Down
7 changes: 5 additions & 2 deletions builder/default.nix
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, buildGHC, fetchurl, writeText, runCommand, pkgconfig, nonReinstallablePkgs }:
{ pkgs, buildPackages, stdenv, lib, haskellLib, ghc, buildGHC, fetchurl, writeText, runCommand, pkgconfig, nonReinstallablePkgs, withPackage }:

{ flags
, package
Expand All @@ -22,6 +22,8 @@
, preInstall
, postInstall

, shellHook

, ...
}@config:

Expand Down Expand Up @@ -67,11 +69,12 @@ let
'';
};

comp-builder = haskellLib.weakCallPackage pkgs ./comp-builder.nix { inherit ghc haskellLib nonReinstallablePkgs; };
comp-builder = haskellLib.weakCallPackage pkgs ./comp-builder.nix { inherit ghc haskellLib nonReinstallablePkgs withPackage; };

buildComp = componentId: component: comp-builder {
inherit componentId component package name src flags setup cabalFile patches revision
preUnpack postUnpack preConfigure postConfigure preBuild postBuild preCheck postCheck preInstall postInstall
shellHook
;
};

Expand Down
77 changes: 77 additions & 0 deletions builder/with-package-wrapper.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# This is a simplified version of the ghcWithPackages wrapper in
# nixpkgs, adapted to work with the package database of a single
# component.
{ lib, stdenv, ghc, runCommand, lndir, makeWrapper
}:

{ package
, configFiles
, postBuild ? ""
}:

let
isGhcjs = ghc.isGhcjs or false;
ghcCommand' = if isGhcjs then "ghcjs" else "ghc";
ghcCommand = "${ghc.targetPrefix}${ghcCommand'}";
ghcCommandCaps= lib.toUpper ghcCommand';
libDir = "$out/lib/${ghcCommand}-${ghc.version}";
docDir = "$out/share/doc/ghc/html";
packageCfgDir = "${libDir}/package.conf.d";

in runCommand "${ghc.name}-with-${package.identifier.name}" {
preferLocalBuild = true;
passthru = {
inherit (ghc) version meta;
baseGhc = ghc;
inherit package;
};
} (
''
. ${makeWrapper}/nix-support/setup-hook

# Start with a ghc...
mkdir -p $out/bin
${lndir}/bin/lndir -silent ${ghc} $out

# ...and replace package database with the one from target package config.
rm -rf ${libDir}
mkdir -p ${libDir}
ln -s ${configFiles}/package.conf.d ${packageCfgDir}

# Wrap compiler executables with correct env variables.
# The NIX_ variables are used by the patched Paths_ghc module.
# The GHC_ENVIRONMENT variable forces ghc to use the build
# dependencies of the component.

for prg in ${ghcCommand} ${ghcCommand}i ${ghcCommand}-${ghc.version} ${ghcCommand}i-${ghc.version} runghc runhaskell; do
if [[ -x "${ghc}/bin/$prg" ]]; then
rm -f $out/bin/$prg
makeWrapper ${ghc}/bin/$prg $out/bin/$prg \
--set "NIX_${ghcCommandCaps}" "$out/bin/${ghcCommand}" \
--set "NIX_${ghcCommandCaps}PKG" "$out/bin/${ghcCommand}-pkg" \
--set "NIX_${ghcCommandCaps}_DOCDIR" "${docDir}" \
--set "NIX_${ghcCommandCaps}_LIBDIR" "${libDir}" \
--set "${ghcCommandCaps}_ENVIRONMENT" "${configFiles}/ghc-environment"
fi
done

# Point ghc-pkg to the package database of the component using the
# GHC_PACKAGE_PATH variable.

for prg in ${ghcCommand}-pkg ${ghcCommand}-pkg-${ghc.version}; do
if [[ -x "${ghc}/bin/$prg" ]]; then
rm -f $out/bin/$prg
makeWrapper ${ghc}/bin/$prg $out/bin/$prg \
--set "${ghcCommandCaps}_PACKAGE_PATH" "${configFiles}/package.conf.d"
fi
done

# fixme: check if this is needed
# haddock was referring to the base ghc, https://github.com/NixOS/nixpkgs/issues/36976
if [[ -x "${ghc}/bin/haddock" ]]; then
rm -f $out/bin/haddock
makeWrapper ${ghc}/bin/haddock $out/bin/haddock \
--set "NIX_${ghcCommandCaps}_LIBDIR" "${libDir}"
fi
''
)
66 changes: 66 additions & 0 deletions doc/removing-with-package-wrapper.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# `ghcWithPackages` wrapper removal

The current [Nixpkgs Haskell infrastructure][nixpkgs-haskell] and `haskell.nix` both
provide a `ghcWithPackages` derivation which contains shell script
wrappers that wrap `ghc` and `ghc-pkg`.

In the Nixpkgs Haskell infrastructure, the wrapper scripts are used
for building Haskell packages. However, in `haskell.nix`, the wrappers
are only used for development environments.

The wrapper scripts provide a `ghc` command that "knows" about the
package set and has all Haskell package dependencies available to it.

We would like to remove the wrapper scripts, but it's currently not
possible to configure all build tools using environment variables
alone.

## Plain `ghc`

When using `ghc` or `ghci` by itself, the `GHC_ENVIRONMENT` variable
can point to a configuration file containing an exact package
set. This works quite well.

## `ghc-pkg`

The package tool `ghc-pkg` does not recognize `GHC_ENVIRONMENT`, but
does recognize a `GHC_PACKAGE_PATH` pointing to a `package.conf.d`.

This works well. However, the `cabal` command will refuse to start if
`GHC_PACKAGE_PATH` is set.

## `Setup.hs`

When invoking `Setup.hs configure`, the package database is provided
with the `--package-db` argument and exact dependencies in the package
set can be provided as `--dependency` arguments.

The `haskell.nix` component builder uses `Setup.hs` with these
command-line options to build Haskell packages.

## `cabal new-build`

Cabal-install will observe the `CABAL_CONFIG` environment variable,
which points to a cabal config file. This config file can provide a
`package-db` value, but it can't specify exact versions of packages.

Cabal is designed to solve dependencies, not simply take the package
set which is given to it.

Therefore, `cabal` does not use `GHC_ENVIRONMENT`, but instead creates
its own environment file. It will not accept `--dependency` arguments.

As far as I know, the best way to force `cabal` to take a pre-computed
package set is to use a `new-freeze` file. However there is no
environment variable (or config file entry) which can specify a path
to a freeze file.

Specifying a `package-db` path in the cabal config file is not enough
for it to successfully resolve dependencies.

As mentioned before, `cabal` does not work when `GHC_PACKAGE_PATH` is
set. The best way to work around this is to wrap `ghc` and `ghc-pkg`
in shell scripts.


[nixpkgs-haskell]: https://nixos.org/nixpkgs/manual/#users-guide-to-the-haskell-infrastructure
Loading