From 5f97eb706a9ddbfd403d1bc9354a51c6866d1787 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Sat, 20 May 2017 17:49:35 +0300 Subject: [PATCH 1/7] shell.nix: bump nixpkgs to tip of NixOS 17.03; adapt to changes --- nixpkgs-src.json | 4 ++-- shell.nix | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nixpkgs-src.json b/nixpkgs-src.json index 879972c7b98..441684719c4 100644 --- a/nixpkgs-src.json +++ b/nixpkgs-src.json @@ -1,6 +1,6 @@ { "owner": "NixOS", "repo": "nixpkgs", - "rev": "464c79ea9f929d1237dbc2df878eedad91767a72", - "sha256": "05kkm978bfai3krsjn6h310xj7q0022bbh7a80kdizglqfkj8krw" + "rev": "3e47e6251930294711d625a8123b7f395dd16b58", + "sha256": "026q9bd4mkk2hwibsss3yh3df483a4ira28vb7zsy54rrx1zy21p" } diff --git a/shell.nix b/shell.nix index 5cc072ba880..247c2017c33 100644 --- a/shell.nix +++ b/shell.nix @@ -16,7 +16,6 @@ ghc = ghcOrig.override (oldArgs: { let parent = (oldArgs.overrides or (_: _: {})) new old; in with new; parent // { intero = overhub old.intero "commercialhaskell/intero" "e546ea086d72b5bf8556727e2983930621c3cb3c" "1qv7l5ri3nysrpmnzfssw8wvdvz0f6bmymnz1agr66fplazid4pn" { doCheck = false; }; - turtle_1_3_0 = doJailbreak old.turtle_1_3_0; }; }); @@ -25,7 +24,7 @@ ghc = ghcOrig.override (oldArgs: { ### drvf = { mkDerivation, stdenv, src ? ./. -, base, turtle_1_3_0, cassava, vector, safe, aeson, yaml, lens-aeson +, base, turtle, cassava, vector, safe, aeson, yaml, lens-aeson }: mkDerivation { pname = "iohk-nixops"; @@ -35,7 +34,7 @@ mkDerivation { isExecutable = true; doHaddock = false; executableHaskellDepends = [ - base turtle_1_3_0 cassava vector safe aeson yaml lens-aeson + base turtle cassava vector safe aeson yaml lens-aeson ]; # description = "Visual mind assistant"; license = stdenv.lib.licenses.agpl3; From 2b41333a6d864c204bbb13262f3e806690d06d18 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Sat, 20 May 2017 17:49:56 +0300 Subject: [PATCH 2/7] shell.nix: slight refactoring --- shell.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/shell.nix b/shell.nix index 247c2017c33..dbf33eac52d 100644 --- a/shell.nix +++ b/shell.nix @@ -1,14 +1,16 @@ -{ pkgs ? import ((import {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./nixpkgs-src.json))) {} -, compiler ? pkgs.haskell.packages.ghc802 +let + nixpkgs = (import {}).fetchFromGitHub (builtins.fromJSON (builtins.readFile ./nixpkgs-src.json)); + pkgs = import nixpkgs {}; +in +{ compiler ? pkgs.haskell.packages.ghc802 , intero ? false }: let ghcOrig = import ./default.nix { inherit pkgs compiler; }; -overcabal = pkgs.haskell.lib.overrideCabal; hubsrc = repo: rev: sha256: pkgs.fetchgit { url = "https://github.com/" + repo; rev = rev; sha256 = sha256; }; -overc = old: args: overcabal old (oldAttrs: (oldAttrs // args)); +overc = old: args: pkgs.haskell.lib.overrideCabal old (oldAttrs: (oldAttrs // args)); overhub = old: repo: rev: sha256: args: overc old ({ src = hubsrc repo rev sha256; } // args); overhage = old: version: sha256: args: overc old ({ version = version; sha256 = sha256; } // args); ghc = ghcOrig.override (oldArgs: { @@ -36,7 +38,6 @@ mkDerivation { executableHaskellDepends = [ base turtle cassava vector safe aeson yaml lens-aeson ]; - # description = "Visual mind assistant"; license = stdenv.lib.licenses.agpl3; }; From 3695407ded2fa6405ddc0b67c3fae1326e306ce6 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Sat, 20 May 2017 17:50:31 +0300 Subject: [PATCH 3/7] shell.nix: attempt to provide NIX_PATH for better development experience --- shell.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/shell.nix b/shell.nix index dbf33eac52d..95c5ed4bd79 100644 --- a/shell.nix +++ b/shell.nix @@ -39,6 +39,12 @@ mkDerivation { base turtle cassava vector safe aeson yaml lens-aeson ]; license = stdenv.lib.licenses.agpl3; + + shellHook = + '' + export NIX_PATH=nixpkgs=${nixpkgs} + echo NIX_PATH=$NIX_PATH + ''; }; drv = (pkgs.haskell.lib.addBuildTools From fc58116057a129875332c548618bca6799a67f82 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Sat, 20 May 2017 19:31:40 +0300 Subject: [PATCH 4/7] shell.nix: add `nix` for the sake of --pure --- shell.nix | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/shell.nix b/shell.nix index 95c5ed4bd79..9120069341e 100644 --- a/shell.nix +++ b/shell.nix @@ -49,10 +49,11 @@ mkDerivation { drv = (pkgs.haskell.lib.addBuildTools (ghc.callPackage drvf { }) -(if intero - then [ pkgs.cabal-install - pkgs.stack - ghc.intero ] - else [])); +((if intero + then [ pkgs.cabal-install + pkgs.stack + ghc.intero ] + else [ ]) ++ + [ pkgs.nix ])); in drv.env From 30369d2fc7b82f159176358b0a95bf41fc8b4920 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Sat, 20 May 2017 19:30:19 +0300 Subject: [PATCH 5/7] [DEVOPS-36] Build speedups in following of https://issues.serokell.io/issue/DEVOPS-36#comment=96-3716 --- default.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 169ba06ff6a..4203b9f8be2 100644 --- a/default.nix +++ b/default.nix @@ -11,6 +11,13 @@ let }); socket-io-src = pkgs.fetchgit (removeAttrs (lib.importJSON ./pkgs/engine-io.json) ["date"]); cardano-sl-src = pkgs.fetchgit (removeAttrs (lib.importJSON ./pkgs/cardano-sl.json) ["date"]); + cores = 4; + threads = 8; + buildSubset = subset: drv: overrideCabal drv (drv: { + buildTarget = subset; + }); + buildSpeedupFlagsMarlowScaling = drv : appendConfigureFlag drv + "--ghc-option=-rtsopts --ghc-option=+RTS --ghc-option=-N${toString threads} --ghc-option=-A128m --ghc-option=-n2m --ghc-option=-qb0 --ghc-option=-qn${toString cores} --ghc-option=-RTS"; in compiler.override { overrides = self: super: { # To generate these go to ./pkgs and run ./generate.sh @@ -62,7 +69,10 @@ in compiler.override { # production full nodes shouldn't use wallet as it means different constants configureFlags = [ "-f-asserts" "-f-dev-mode" "-fwith-explorer"]; }); - cardano-sl-static = justStaticExecutables self.cardano-sl; + cardano-sl-static = + buildSpeedupFlagsMarlowScaling + (buildSubset ("cardano-node") + (dontCheck (linkWithGold (justStaticExecutables self.cardano-sl)))); cardano-report-server-static = justStaticExecutables self.cardano-report-server; cardano-sl-explorer-static = justStaticExecutables self.cardano-sl-explorer; From 9927c7711ec6d406dc23faed5de9f6114f09adcf Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Mon, 22 May 2017 13:49:01 +0300 Subject: [PATCH 6/7] [DEVOPS-36] review: add `builder-threads` parameter to default.nix (default to 8) --- default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index 4203b9f8be2..6b4e86cf168 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,6 @@ { pkgs ? (import {}) , compiler ? pkgs.haskell.packages.ghc802 +, builder-threads ? 8 }: with (import { inherit pkgs;}); @@ -11,13 +12,13 @@ let }); socket-io-src = pkgs.fetchgit (removeAttrs (lib.importJSON ./pkgs/engine-io.json) ["date"]); cardano-sl-src = pkgs.fetchgit (removeAttrs (lib.importJSON ./pkgs/cardano-sl.json) ["date"]); - cores = 4; - threads = 8; + + builder-cores = builtins.div builder-threads 2; buildSubset = subset: drv: overrideCabal drv (drv: { buildTarget = subset; }); buildSpeedupFlagsMarlowScaling = drv : appendConfigureFlag drv - "--ghc-option=-rtsopts --ghc-option=+RTS --ghc-option=-N${toString threads} --ghc-option=-A128m --ghc-option=-n2m --ghc-option=-qb0 --ghc-option=-qn${toString cores} --ghc-option=-RTS"; + "--ghc-option=-rtsopts --ghc-option=+RTS --ghc-option=-N${toString builder-threads} --ghc-option=-A128m --ghc-option=-n2m --ghc-option=-qb0 --ghc-option=-qn${toString builder-cores} --ghc-option=-RTS"; in compiler.override { overrides = self: super: { # To generate these go to ./pkgs and run ./generate.sh From 041f66894605c661e3fc6bc17403c02129f25217 Mon Sep 17 00:00:00 2001 From: Kosyrev Serge Date: Mon, 22 May 2017 17:04:03 +0300 Subject: [PATCH 7/7] [DEVOPS-36] disable build subsetting, due to a cabal limitation Details: https://factisresearch.blogspot.ru/2010/09/speeding-up-your-cabal-builds-part-ii.html --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 6b4e86cf168..d9520904695 100644 --- a/default.nix +++ b/default.nix @@ -72,7 +72,7 @@ in compiler.override { }); cardano-sl-static = buildSpeedupFlagsMarlowScaling - (buildSubset ("cardano-node") + (# buildSubset ("cardano-node") -- unfortunately, we need customizing the .cabal file for this to work.. (dontCheck (linkWithGold (justStaticExecutables self.cardano-sl)))); cardano-report-server-static = justStaticExecutables self.cardano-report-server; cardano-sl-explorer-static = justStaticExecutables self.cardano-sl-explorer;