Better error handling in restore benchmark

.buildkite/bench-restore.sh

@@ -14,21 +14,22 @@ log=restore.log
 results=restore-$network.txt
 total_time=restore-time.txt
 
-: "${NODE_DB:=$HOME/node-db-$network}"
+if [ -n "${SCRATCH_DIR:-}" ]; then
+  mkdir -pv "$SCRATCH_DIR"
+  export TMPDIR="$SCRATCH_DIR/tmp"
+  mkdir -pv "$TMPDIR"
+fi
+
+: "${node_db:=$HOME/node-db-$network}"
 
 echo "--- Build"
 
 nix-build -A benchmarks.cardano-wallet.restore -o bench-restore
 
-bench="./bench-restore/bin/restore $network"
+bench="./bench-restore/bin/restore $network --node-db $node_db"
 
 echo "--- Run benchmarks - $network"
 
-if [ -n "${SCRATCH_DIR:-}" ]; then
-  mkdir -p "$SCRATCH_DIR"
-  export HOME="$SCRATCH_DIR"
-fi
-
 command time -o $total_time -v $bench +RTS -N2 -qg -A1m -I0 -T -M8G -h -RTS 2>&1 | tee $log
 
 grep -v INFO $log | awk '/All results/,EOF { print $0 }' > $results

.buildkite/nightly.yml

@@ -1,8 +1,5 @@
 env:
-  NIX_PATH: "channel:nixos-19.03"
-  BUILD_DIR: "/build/cardano-wallet"
-  STACK_ROOT: "/build/cardano-wallet.stack"
-  CACHE_DIR: "/cache/cardano-wallet"
+  NIX_PATH: "channel:nixos-20.09"
   SCRATCH_DIR: "/scratch/cardano-wallet"
 
 steps:

.buildkite/rebuild.hs

@@ -155,6 +155,7 @@ buildStep dryRun bk nightly = do
     build opt args =
         run dryRun "stack" $ concat
             [ color "always"
+            , [ "--no-terminal" ]
             , [ "build" ]
             , [ "--bench" ]
             , [ "--no-run-benchmarks" ]
@@ -168,6 +169,7 @@ buildStep dryRun bk nightly = do
     test opt args =
         run dryRun "stack" $ concat
             [ color "always"
+            , [ "--no-terminal" ]
             , [ "test" ]
             , fast opt
             , case qaLevel nightly bk of

lib/core-integration/src/Cardano/Wallet/BenchShared.hs

@@ -52,7 +52,7 @@ import Cardano.Wallet.Network.Ports
 import Control.DeepSeq
     ( NFData, rnf )
 import Control.Monad
-    ( forM, void )
+    ( forM )
 import Criterion.Measurement
     ( getTime, initializeTime, secs )
 import Data.Aeson
@@ -63,6 +63,8 @@ import Data.Maybe
     ( fromMaybe )
 import Data.Text
     ( Text )
+import Data.Text.Class
+    ( ToText (..) )
 import Fmt
     ( Buildable (..), nameF, pretty )
 import GHC.Generics
@@ -94,7 +96,7 @@ import System.Directory
 import System.Environment
     ( lookupEnv )
 import System.Exit
-    ( die )
+    ( ExitCode (..), die )
 import System.FilePath
     ( (</>) )
 import System.IO
@@ -118,7 +120,7 @@ execBenchWithNode
     -- ^ Get backend-specific network configuration from args
     -> (Trace IO Text -> cfg -> CardanoNodeConn -> IO ())
     -- ^ Action to run
-    -> IO ()
+    -> IO ExitCode
 execBenchWithNode networkConfig action = do
     args <- getRestoreBenchArgs
 
@@ -130,12 +132,18 @@ execBenchWithNode networkConfig action = do
     installSignalHandlers (return ())
 
     case argUseAlreadyRunningNodeSocketPath args of
-        Just conn ->
+        Just conn -> do
             action tr (networkConfig args) conn
+            pure ExitSuccess
         Nothing -> do
-            void $ withNetworkConfiguration args $ \nodeConfig ->
+            res <- withNetworkConfiguration args $ \nodeConfig ->
                 withCardanoNode (trMessageText tr) nodeConfig $
                     action tr (networkConfig args)
+            case res of
+                Left exited -> do
+                    sayErr $ "FAIL: cardano-node exited with status " <> toText exited
+                    pure $ ExitFailure 1
+                Right _ -> pure ExitSuccess
 
 withNetworkConfiguration :: RestoreBenchArgs -> (CardanoNodeConfig -> IO a) -> IO a
 withNetworkConfiguration args action = do

lib/launcher/src/Cardano/Launcher.hs

@@ -347,6 +347,12 @@ instance HasSeverityAnnotation WaitForProcessLog where
         MsgWaitAfter _ -> Debug
         MsgWaitCancelled -> Debug
 
+instance ToText ProcessHasExited where
+    toText (ProcessHasExited name code) =
+        "Child process "+|name|+" exited with "+|statusText code|+""
+    toText (ProcessDidNotStart name _e) =
+        "Could not start "+|name|+""
+
 instance ToText LauncherLog where
     toText ll = fmt $ case ll of
         MsgLauncherStart cmd args ->
@@ -355,10 +361,7 @@ instance ToText LauncherLog where
             "["+|name|+"."+|pid|+"] "+|toText msg|+""
         MsgLauncherFinish Nothing ->
             "Action finished"
-        MsgLauncherFinish (Just (ProcessDidNotStart name _e)) ->
-            "Could not start "+|name|+""
-        MsgLauncherFinish (Just (ProcessHasExited name code)) ->
-            "Child process "+|name|+" exited with "+|statusText code|+""
+        MsgLauncherFinish (Just exited) -> build $ toText exited
         MsgLauncherCleanup ->
             "Begin process cleanup"
         MsgLauncherCleanupTimedOut t ->

lib/shelley/bench/Restore.hs

@@ -194,6 +194,8 @@ import Numeric
     ( showFFloat )
 import Say
     ( sayErr )
+import System.Exit
+    ( exitWith )
 import System.FilePath
     ( (</>) )
 import System.IO
@@ -220,7 +222,7 @@ import qualified Data.Text as T
 import qualified Data.Text.Encoding as T
 
 main :: IO ()
-main = execBenchWithNode argsNetworkConfig cardanoRestoreBench
+main = execBenchWithNode argsNetworkConfig cardanoRestoreBench >>= exitWith
 
 {-------------------------------------------------------------------------------
                                 Shelley benchmarks