Merge #888

888: Update Haskell.nix r=KtorZ a=rvl Relates to #703. # Overview - [x] Updates the Haskell.nix version so that we can build for windows again. - [x] Keeps more or less the same structure as before - i.e. no IFD, no niv. I don't want too much change at once. - [x] Does however update nixpkgs from iohk fork of 18.09 → upstream 19.03. - [x] Provides a big bundle of scripts and tests that can be run on Windows. [Hydra jobset](https://hydra.iohk.io/jobset/Cardano/cardano-wallet-pr-888) [Hercules jobset](https://hercules-ci.com/github/input-output-hk/cardano-wallet) [Link to download Windows testing bundle for PR #888](https://hydra.iohk.io/job/Cardano/cardano-wallet-pr-888/cardano-wallet-jormungandr-win64/latest) [Link to download Windows testing bundle (master branch)](https://hydra.iohk.io/job/Cardano/cardano-wallet/cardano-wallet-jormungandr-win64/latest) # Comments @angerman Please help - [x] Fix failure for missing `hsc2hs` - [x] Fix a previous mistake that affects cross builds (failing to cross-compile bash for some reason) - [x] Worked around an [evaluation error](https://hydra.iohk.io/jobset/Cardano/cardano-wallet-pr-888#tabs-errors) because weeder and ghci are no longer available from `pkgs.haskellPackages`. - [ ] Evaluation is taking a rather long time (5+ hours) in Hydra. - [ ] Some of the Buildkite pipeline steps don't work with the updated Haskell.nix. - [ ] Tests for windows are built not not executed. The tests should be run under wine. # Next PRs - [ ] Fix failures in sqlite tests on Windows ⇒ #918. - [ ] Fix failure in launch command on Windows. - [ ] Fix the UTF-8 text issues on Windows ⇒ #914. - [ ] Add code which tells the windows command prompt to interpret ANSI color codes. - [ ] Identify other test failures and fix. Co-authored-by: Rodney Lorrimar <[email protected]> Co-authored-by: Michael Bishop <[email protected]>
cardano-foundation · Nov 4, 2019 · f75f38d · f75f38d
2 parents 3e8c756 + d27da42
commit f75f38d
Show file tree

Hide file tree

Showing 13 changed files with 163 additions and 54 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -21,12 +21,12 @@ steps:
       system: x86_64-linux
 
   - label: 'Check Stylish Haskell'
-    command: 'nix-shell --run .buildkite/check-stylish.sh'
+    command: 'nix-shell -A ci-shell --run .buildkite/check-stylish.sh'
     agents:
       system: x86_64-linux
 
   - label: 'HLint'
-    command: 'nix-shell --run "hlint lib"'
+    command: 'nix-shell -A ci-shell --run "hlint lib"'
     agents:
       system: x86_64-linux
 

diff --git a/default.nix b/default.nix
@@ -3,23 +3,20 @@
 , config ? {}
 # Import IOHK common nix lib
 , iohkLib ? import ./nix/iohk-common.nix { inherit system crossSystem config; }
-# Use nixpkgs pin from iohkLib
-, pkgs ? iohkLib.pkgs
+# Use pinned Nixpkgs with Haskell.nix overlay
+, pkgs ? import ./nix/nixpkgs-haskell.nix  { inherit system crossSystem config; }
 }:
 
 with import ./nix/util.nix { inherit pkgs; };
 
 let
-  haskell = iohkLib.nix-tools.haskell { inherit pkgs; };
-  src = iohkLib.cleanSourceHaskell ./.;
+  src = pkgs.haskell-nix.cleanSourceHaskell ./.;
 
-  jmPkgs = import ./nix/jormungandr.nix { inherit iohkLib pkgs; };
+  jmPkgs = import ./nix/jormungandr.nix { inherit iohkLib; };
   inherit (jmPkgs) jormungandr jormungandr-cli;
 
   haskellPackages = import ./nix/default.nix {
-    inherit pkgs haskell src;
-    inherit jmPkgs;
-    inherit (iohkLib.nix-tools) iohk-extras iohk-module;
+    inherit pkgs src jmPkgs;
   };
 
   inherit (haskellPackages.cardano-wallet-core.identifier) version;
@@ -48,10 +45,9 @@ in {
       bech32
       text-class
     ];
-    buildInputs =
-      with pkgs.haskellPackages; [ stylish-haskell weeder ghcid ]
+    buildInputs = (with pkgs.haskellPackages; [ stylish-haskell weeder ghcid ])
+      ++ (with iohkLib; [ hlint openapi-spec-validator ])
       ++ [ jormungandr jormungandr-cli
-           pkgs.pkgconfig pkgs.sqlite-interactive
-           iohkLib.hlint iohkLib.openapi-spec-validator ];
+           pkgs.pkgconfig pkgs.sqlite-interactive ];
   };
 }
diff --git a/nix/ci.nix b/nix/ci.nix
@@ -2,18 +2,35 @@
 # https://hercules-ci.com/github/input-output-hk/cardano-wallet
 # https://docs.hercules-ci.com/hercules-ci/getting-started/minimal-repository/
 
-builtins.mapAttrs (system: _:
-  let
-    walletPkgs = import ../default.nix { inherit system; };
-  in
+let
+  inherit (import ./nixpkgs-haskell.nix {}) lib;
+
+  walletJobs = walletPkgs:
     walletPkgs.pkgs.recurseIntoAttrs {
       inherit (walletPkgs)
         cardano-wallet-jormungandr
         tests
         benchmarks;
-    }
-) {
-  x86_64-linux = {};
-  # Uncomment to test build on macOS too
-  # x86_64-darwin = {};
-}
+    };
+
+  nativeBuilds = builtins.mapAttrs (system: _:
+    walletJobs (import ../default.nix { inherit system; })
+  ) {
+    x86_64-linux = {};
+    # Uncomment to test build on macOS too
+    # x86_64-darwin = {};
+  };
+
+  crossSystems = with lib.systems.examples;
+    [ mingwW64 ];
+
+  crossBuilds = lib.listToAttrs (map (crossSystem: {
+    name = crossSystem.config;
+    value = walletJobs (import ../default.nix {
+      system = "x86_64-linux";
+      inherit crossSystem;
+    });
+  }) crossSystems);
+
+in
+  nativeBuilds // crossBuilds
diff --git a/nix/default.nix b/nix/default.nix
@@ -1,21 +1,15 @@
 { pkgs
 
-# haskell.nix
-, haskell
-
 # Filtered sources of this project
 , src
 
 # Dependencies of cardano-wallet-jormungandr
 , jmPkgs
-
-# Customisations for cross-compiling
-, iohk-extras ? {}
-, iohk-module ? {}
-
 }:
 
 let
+  haskell = pkgs.haskell-nix;
+
   # our packages
   stack-pkgs = import ./.stack.nix/default.nix;
 
@@ -87,15 +81,14 @@ let
         # Katip has Win32 (>=2.3 && <2.6) constraint
         packages.katip.doExactConfig = true;
       }
-
-      # The iohk-module will supply us with the necessary
-      # cross compilation plumbing to make Template Haskell
-      # work when cross compiling.
-      iohk-module
     ];
     pkg-def-extras = [
-      # Use the iohk-extras patched GHC for cross-compiling.
-      iohk-extras.${compiler}
+      # Workaround for https://github.com/input-output-hk/haskell.nix/issues/214
+      (hackage: {
+        packages = {
+          "hsc2hs" = (((hackage.hsc2hs)."0.68.4").revisions).default;
+        };
+      })
     ];
   };
 

diff --git a/nix/dep.nix b/nix/dep.nix
@@ -0,0 +1,11 @@
+# Provides a function for fetching a GitHub repo from a JSON spec.
+{ name, specJSON }:
+
+let
+  spec = builtins.fromJSON (builtins.readFile specJSON);
+in
+  builtins.fetchTarball {
+    inherit name;
+    url = "${spec.url}/archive/${spec.rev}.tar.gz";
+    inherit (spec) sha256;
+  }
diff --git a/nix/haskell-nix-src.json b/nix/haskell-nix-src.json
@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/input-output-hk/haskell.nix",
+  "rev": "fcba9447b31a0802fab15f46fa18d6b9675ab528",
+  "date": "2019-11-02T21:33:00+13:00",
+  "sha256": "0s7gdcjxv8y7bmbi7wd5cp8jlvdcn2bgf8kaqyy25x0bqcdafaz3",
+  "fetchSubmodules": false
+}
diff --git a/nix/nixpkgs-haskell.nix b/nix/nixpkgs-haskell.nix
@@ -0,0 +1,49 @@
+############################################################################
+# Nixpkgs 19.03 with Haskell.nix overlay
+#
+# This works by importing the base Nixpkgs with extra config and
+# overlays provided by Haskell.nix. It will contain patched GHCs
+# suitable for cross-compiling to Windows, and the `haskell-nix`
+# component builder.
+#
+# To update the Nixpkgs version, use:
+#
+#     nix-prefetch-git https://github.com/NixOS/nixpkgs-channels refs/heads/nixpkgs-19.03-darwin | tee nix/nixpkgs-src.json
+#
+# This will pick the latest revision on the 19.03 branch.
+#
+# To update the Haskell.nix version, use:
+#
+#     nix-prefetch-git https://github.com/input-output-hk/haskell.nix | tee nix/haskell-nix-src.json
+#
+############################################################################
+
+# Arguments to pass when importing nixpkgs.
+{ system ? builtins.currentSystem
+, crossSystem ? null
+, config ? {}
+}:
+
+let
+  haskell-nix-src = import ./dep.nix {
+    name = "haskell.nix";
+    specJSON = ./haskell-nix-src.json;
+  };
+  nixpkgs = import ./dep.nix {
+    name = "nixpkgs";
+    specJSON = ./nixpkgs-src.json;
+  };
+
+  haskellNixArgs = import haskell-nix-src;
+
+  # Merge config and overlays provided by Haskell.nix into
+  # our own nixpkgs args.
+  args = haskellNixArgs // {
+    inherit system crossSystem;
+    config = (haskellNixArgs.config or {}) // config;
+    overlays = (haskellNixArgs.overlays or []) ++ overlays;
+  };
+
+  overlays = [];
+in
+  import nixpkgs args
diff --git a/nix/nixpkgs-src.json b/nix/nixpkgs-src.json
@@ -0,0 +1,7 @@
+{
+  "url": "https://github.com/NixOS/nixpkgs",
+  "rev": "a8f81dc037a5977414a356dd068f2621b3c89b60",
+  "date": "2019-10-13T18:41:59-04:00",
+  "sha256": "01z13axll5g5yl00lz9adr2jw2bs12g9skhbb1vxy8p7fjjbhhhm",
+  "fetchSubmodules": false
+}
diff --git a/nix/package-jormungandr.nix b/nix/package-jormungandr.nix
@@ -9,16 +9,17 @@
 { pkgs
 , version
 , cardano-wallet-jormungandr
-, jmPkgs ? import ./jormungandr.nix { inherit pkgs; }
-, jormungandr ? jmPkgs.jormungandr
-, jormungandr-win64 ? jmPkgs.jormungandr-win64
+, jmPkgs
 }:
 
 with pkgs.lib;
 
 let
   name = "cardano-wallet-jormungandr-${version}";
 
+  jormungandr = jmPkgs.jormungandr;
+  jormungandr-win64 = jmPkgs.jormungandr-win64;
+
   deps = {
     nix = ''
       strip $out/bin/cardano-wallet-jormungandr
@@ -30,7 +31,7 @@ let
     '';
     windows = ''
       cp -v ${pkgs.libffi}/bin/libffi-6.dll $out/bin
-      cp ${jormungandr-win64}/* $out/bin
+      cp ${jormungandr-win64}/bin/* $out/bin
     '';
   };
   provideDeps = { nix, darwin ? "", windows ? "" }:
@@ -39,7 +40,7 @@ let
 
 in pkgs.runCommand name {
   inherit version;
-  nativeBuildInputs = [ pkgs.makeWrapper pkgs.binutils ];
+  nativeBuildInputs = [ pkgs.buildPackages.makeWrapper pkgs.buildPackages.binutils ];
 } ''
   cp -R ${cardano-wallet-jormungandr} $out
   chmod -R +w $out

diff --git a/nix/util.nix b/nix/util.nix
@@ -7,10 +7,5 @@ with pkgs.lib;
     (hasPrefix "cardano-wallet" package.identifier.name) ||
     (elem package.identifier.name [ "text-class" "bech32" ]);
 
-  # TODO: use upstreamed version:
-  # https://github.com/input-output-hk/haskell.nix/pull/224
-  collectComponents = group: packageSel: haskellPackages:
-    (mapAttrs (_: package: package.components.${group} // { recurseForDerivations = true; })
-     (filterAttrs (name: package: (package.isHaskell or false) && packageSel package) haskellPackages))
-    // { recurseForDerivations = true; };
+  inherit (pkgs.haskell-nix.haskellLib) collectComponents;
 }
diff --git a/nix/windows-release.nix b/nix/windows-release.nix
@@ -8,8 +8,10 @@
 ############################################################################
 
 { pkgs
-, cardano-wallet-jormungandr
 , project
+, cardano-wallet-jormungandr
+, tests ? []
+, benchmarks ? []
 }:
 
 let
@@ -19,10 +21,15 @@ let
     jormungandr.exe --config config.yaml --genesis-block block0.bin --secret secret.yaml
   '';
   cw-bat = pkgs.writeText "cw.bat" ''
-    cardano-wallet-jormungandr.exe serve --node-port 8081 --genesis-hash HASH --database c:\\cardano-wallet-jormungandr\\wallet.db
+    cardano-wallet-jormungandr.exe serve --node-port 8080 --genesis-block-hash HASH --database c:\\cardano-wallet-jormungandr\\wallets
+  '';
+  launch-bat = pkgs.writeText "launch.bat" ''
+    cardano-wallet-jormungandr.exe launch --genesis-block-hash HASH --state-dir c:\\cardano-wallet-jormungandr %*
   '';
+
 in pkgs.runCommand name {
   nativeBuildInputs = [ pkgs.zip pkgs.jq pkgs.gnused project.jormungandr-cli ];
+  passthru = { inherit tests benchmarks; };
 } ''
   mkdir -pv jm $out/nix-support
   cd jm
@@ -32,8 +39,19 @@ in pkgs.runCommand name {
   cp -v ${jm-bat} jm.bat
   hash="$(jcli genesis hash --input block0.bin)"
   sed -e "s/HASH/$hash/" ${cw-bat} > cw.bat
-  sed -e 's/storage:.*/storage: "c:\\\\cardano-wallet-jormungandr\\\\storage"/' \
+  sed -e "s/HASH/$hash/" ${launch-bat} > launch.bat
+  sed -e 's/storage:.*/storage: "c:\\\\cardano-wallet-jormungandr\\\\chain"/' \
       ${testData}/config.yaml > config.yaml
+
+  ${pkgs.lib.concatMapStringsSep "\n" (test: ''
+    pkg=`ls -1 ${test}`
+    exe=`cd ${test}; ls -1 $pkg`
+    name=$pkg-test-$exe
+    cp ${test}/$pkg/$exe $name
+    echo $name >> tests.bat
+    echo "if %errorlevel% neq 0 exit /b %errorlevel%" >> tests.bat
+  '') tests}
+
   chmod -R +w .
 
   zip -r $out/${name}.zip .

diff --git a/release.nix b/release.nix
@@ -25,6 +25,12 @@ let
     "${mingwW64.config}" = mapTestOnCross mingwW64 (packagePlatformsCross project);
   }
   // {
+    ci-tools = {
+      inherit (import ./nix/iohk-common.nix {}) hlint;
+      inherit ((import ./nix/nixpkgs-haskell.nix {}).haskellPackages) stylish-haskell;
+    };
+    inherit ((import ./. {}).pkgs.haskell-nix) haskellNixRoots;
+
     # This aggregate job is what IOHK Hydra uses to update
     # the CI status in GitHub.
     required = mkRequiredJob (
@@ -39,6 +45,8 @@ let
     cardano-wallet-jormungandr-win64 = import ./nix/windows-release.nix {
       inherit pkgs project;
       cardano-wallet-jormungandr = jobs.x86_64-pc-mingw32.cardano-wallet-jormungandr.x86_64-linux;
+      tests = collectTests jobs.x86_64-pc-mingw32.tests;
+      benchmarks = collectTests jobs.x86_64-pc-mingw32.benchmarks;
     };
 
     # These derivations are used for the Daedalus installer.

diff --git a/shell.nix b/shell.nix
@@ -1 +1,8 @@
-(import ./default.nix {}).shell
+let
+  self = import ./default.nix {};
+  # a shell that only has basic CI tools, and no dependencies
+  ci-shell = self.pkgs.stdenv.mkDerivation {
+    name = "ci-shell";
+    buildInputs = [ self.pkgs.haskellPackages.stylish-haskell self.iohkLib.hlint ];
+  };
+in self.shell // { inherit ci-shell; }