Merge #2568

2568: Better error handling in restore benchmark r=Anviking a=rvl

### Issue Number

ADP-804

### Overview

- Handle case where node exits before restore bench is finished.
- Fix disk space error.

### Comments

[Buildkite nightly builds](https://buildkite.com/input-output-hk/cardano-wallet-nightly/builds?branch=rvl/adp-804/fix-restore-bench)

The restore benchmark no longer fails from the original error. It now seems to reach the 10 hour timeout on mainnet. So the benchmark wallets need to be made smaller... in another PR.


Co-authored-by: Rodney Lorrimar <[email protected]>

.buildkite/bench-restore.sh

@@ -14,21 +14,22 @@ log=restore.log
 results=restore-$network.txt
 total_time=restore-time.txt
 
-: "${NODE_DB:=$HOME/node-db-$network}"
+if [ -n "${SCRATCH_DIR:-}" ]; then
+  mkdir -pv "$SCRATCH_DIR"
+  export TMPDIR="$SCRATCH_DIR/tmp"
+  mkdir -pv "$TMPDIR"
+fi
+
+: "${node_db:=$HOME/node-db-$network}"
 
 echo "--- Build"
 
 nix-build -A benchmarks.cardano-wallet.restore -o bench-restore
 
-bench="./bench-restore/bin/restore $network"
+bench="./bench-restore/bin/restore $network --node-db $node_db"
 
 echo "--- Run benchmarks - $network"
 
-if [ -n "${SCRATCH_DIR:-}" ]; then
-  mkdir -p "$SCRATCH_DIR"
-  export HOME="$SCRATCH_DIR"
-fi
-
 command time -o $total_time -v $bench +RTS -N2 -qg -A1m -I0 -T -M8G -h -RTS 2>&1 | tee $log
 
 grep -v INFO $log | awk '/All results/,EOF { print $0 }' > $results

.buildkite/nightly.yml

@@ -1,8 +1,5 @@
 env:
-  NIX_PATH: "channel:nixos-19.03"
-  BUILD_DIR: "/build/cardano-wallet"
-  STACK_ROOT: "/build/cardano-wallet.stack"
-  CACHE_DIR: "/cache/cardano-wallet"
+  NIX_PATH: "channel:nixos-20.09"
   SCRATCH_DIR: "/scratch/cardano-wallet"
 
 steps:

.buildkite/rebuild.hs

@@ -155,6 +155,7 @@ buildStep dryRun bk nightly = do
     build opt args =
         run dryRun "stack" $ concat
             [ color "always"
+            , [ "--no-terminal" ]
             , [ "build" ]
             , [ "--bench" ]
             , [ "--no-run-benchmarks" ]
@@ -168,6 +169,7 @@ buildStep dryRun bk nightly = do
     test opt args =
         run dryRun "stack" $ concat
             [ color "always"
+            , [ "--no-terminal" ]
             , [ "test" ]
             , fast opt
             , case qaLevel nightly bk of

lib/core-integration/src/Cardano/Wallet/BenchShared.hs

@@ -52,7 +52,7 @@ import Cardano.Wallet.Network.Ports
 import Control.DeepSeq
     ( NFData, rnf )
 import Control.Monad
-    ( forM, void )
+    ( forM )
 import Criterion.Measurement
     ( getTime, initializeTime, secs )
 import Data.Aeson
@@ -63,6 +63,8 @@ import Data.Maybe
     ( fromMaybe )
 import Data.Text
     ( Text )
+import Data.Text.Class
+    ( ToText (..) )
 import Fmt
     ( Buildable (..), nameF, pretty )
 import GHC.Generics
@@ -94,7 +96,7 @@ import System.Directory
 import System.Environment
     ( lookupEnv )
 import System.Exit
-    ( die )
+    ( ExitCode (..), die )
 import System.FilePath
     ( (</>) )
 import System.IO
@@ -118,7 +120,7 @@ execBenchWithNode
     -- ^ Get backend-specific network configuration from args
     -> (Trace IO Text -> cfg -> CardanoNodeConn -> IO ())
     -- ^ Action to run
-    -> IO ()
+    -> IO ExitCode
 execBenchWithNode networkConfig action = do
     args <- getRestoreBenchArgs
 
@@ -130,12 +132,18 @@ execBenchWithNode networkConfig action = do
     installSignalHandlers (return ())
 
     case argUseAlreadyRunningNodeSocketPath args of
-        Just conn ->
+        Just conn -> do
             action tr (networkConfig args) conn
+            pure ExitSuccess
         Nothing -> do
-            void $ withNetworkConfiguration args $ \nodeConfig ->
+            res <- withNetworkConfiguration args $ \nodeConfig ->
                 withCardanoNode (trMessageText tr) nodeConfig $
                     action tr (networkConfig args)
+            case res of
+                Left exited -> do
+                    sayErr $ "FAIL: cardano-node exited with status " <> toText exited
+                    pure $ ExitFailure 1
+                Right _ -> pure ExitSuccess
 
 withNetworkConfiguration :: RestoreBenchArgs -> (CardanoNodeConfig -> IO a) -> IO a
 withNetworkConfiguration args action = do

lib/launcher/src/Cardano/Launcher.hs

@@ -347,6 +347,12 @@ instance HasSeverityAnnotation WaitForProcessLog where
         MsgWaitAfter _ -> Debug
         MsgWaitCancelled -> Debug
 
+instance ToText ProcessHasExited where
+    toText (ProcessHasExited name code) =
+        "Child process "+|name|+" exited with "+|statusText code|+""
+    toText (ProcessDidNotStart name _e) =
+        "Could not start "+|name|+""
+
 instance ToText LauncherLog where
     toText ll = fmt $ case ll of
         MsgLauncherStart cmd args ->
@@ -355,10 +361,7 @@ instance ToText LauncherLog where
             "["+|name|+"."+|pid|+"] "+|toText msg|+""
         MsgLauncherFinish Nothing ->
             "Action finished"
-        MsgLauncherFinish (Just (ProcessDidNotStart name _e)) ->
-            "Could not start "+|name|+""
-        MsgLauncherFinish (Just (ProcessHasExited name code)) ->
-            "Child process "+|name|+" exited with "+|statusText code|+""
+        MsgLauncherFinish (Just exited) -> build $ toText exited
         MsgLauncherCleanup ->
             "Begin process cleanup"
         MsgLauncherCleanupTimedOut t ->

lib/shelley/bench/Restore.hs

@@ -194,6 +194,8 @@ import Numeric
     ( showFFloat )
 import Say
     ( sayErr )
+import System.Exit
+    ( exitWith )
 import System.FilePath
     ( (</>) )
 import System.IO
@@ -220,7 +222,7 @@ import qualified Data.Text as T
 import qualified Data.Text.Encoding as T
 
 main :: IO ()
-main = execBenchWithNode argsNetworkConfig cardanoRestoreBench
+main = execBenchWithNode argsNetworkConfig cardanoRestoreBench >>= exitWith
 
 {-------------------------------------------------------------------------------
                                 Shelley benchmarks