From 6369e792f442cb87ecd5e109d99bbe814c2ef722 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Thu, 18 Mar 2021 16:46:05 +1000 Subject: [PATCH 1/4] Try to have less terminal codes in the buildkite log --- .buildkite/rebuild.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/rebuild.hs b/.buildkite/rebuild.hs index c86e91a0aeb..89b6b9cbd41 100644 --- a/.buildkite/rebuild.hs +++ b/.buildkite/rebuild.hs @@ -155,6 +155,7 @@ buildStep dryRun bk nightly = do build opt args = run dryRun "stack" $ concat [ color "always" + , [ "--no-terminal" ] , [ "build" ] , [ "--bench" ] , [ "--no-run-benchmarks" ] @@ -168,6 +169,7 @@ buildStep dryRun bk nightly = do test opt args = run dryRun "stack" $ concat [ color "always" + , [ "--no-terminal" ] , [ "test" ] , fast opt , case qaLevel nightly bk of From fd9168118e4d29b72eb7fc02caa3a1489a66e9b9 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Thu, 18 Mar 2021 16:53:33 +1000 Subject: [PATCH 2/4] Handle case where node exits before restore bench is finished --- .../src/Cardano/Wallet/BenchShared.hs | 18 +++++++++++++----- lib/launcher/src/Cardano/Launcher.hs | 11 +++++++---- lib/shelley/bench/Restore.hs | 4 +++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/core-integration/src/Cardano/Wallet/BenchShared.hs b/lib/core-integration/src/Cardano/Wallet/BenchShared.hs index 43836a38031..94c31fc808d 100644 --- a/lib/core-integration/src/Cardano/Wallet/BenchShared.hs +++ b/lib/core-integration/src/Cardano/Wallet/BenchShared.hs @@ -52,7 +52,7 @@ import Cardano.Wallet.Network.Ports import Control.DeepSeq ( NFData, rnf ) import Control.Monad - ( forM, void ) + ( forM ) import Criterion.Measurement ( getTime, initializeTime, secs ) import Data.Aeson @@ -63,6 +63,8 @@ import Data.Maybe ( fromMaybe ) import Data.Text ( Text ) +import Data.Text.Class + ( ToText (..) ) import Fmt ( Buildable (..), nameF, pretty ) import GHC.Generics @@ -94,7 +96,7 @@ import System.Directory import System.Environment ( lookupEnv ) import System.Exit - ( die ) + ( ExitCode (..), die ) import System.FilePath ( () ) import System.IO @@ -118,7 +120,7 @@ execBenchWithNode -- ^ Get backend-specific network configuration from args -> (Trace IO Text -> cfg -> CardanoNodeConn -> IO ()) -- ^ Action to run - -> IO () + -> IO ExitCode execBenchWithNode networkConfig action = do args <- getRestoreBenchArgs @@ -130,12 +132,18 @@ execBenchWithNode networkConfig action = do installSignalHandlers (return ()) case argUseAlreadyRunningNodeSocketPath args of - Just conn -> + Just conn -> do action tr (networkConfig args) conn + pure ExitSuccess Nothing -> do - void $ withNetworkConfiguration args $ \nodeConfig -> + res <- withNetworkConfiguration args $ \nodeConfig -> withCardanoNode (trMessageText tr) nodeConfig $ action tr (networkConfig args) + case res of + Left exited -> do + sayErr $ "FAIL: cardano-node exited with status " <> toText exited + pure $ ExitFailure 1 + Right _ -> pure ExitSuccess withNetworkConfiguration :: RestoreBenchArgs -> (CardanoNodeConfig -> IO a) -> IO a withNetworkConfiguration args action = do diff --git a/lib/launcher/src/Cardano/Launcher.hs b/lib/launcher/src/Cardano/Launcher.hs index e70a9bff68c..bb99a75dee8 100644 --- a/lib/launcher/src/Cardano/Launcher.hs +++ b/lib/launcher/src/Cardano/Launcher.hs @@ -347,6 +347,12 @@ instance HasSeverityAnnotation WaitForProcessLog where MsgWaitAfter _ -> Debug MsgWaitCancelled -> Debug +instance ToText ProcessHasExited where + toText (ProcessHasExited name code) = + "Child process "+|name|+" exited with "+|statusText code|+"" + toText (ProcessDidNotStart name _e) = + "Could not start "+|name|+"" + instance ToText LauncherLog where toText ll = fmt $ case ll of MsgLauncherStart cmd args -> @@ -355,10 +361,7 @@ instance ToText LauncherLog where "["+|name|+"."+|pid|+"] "+|toText msg|+"" MsgLauncherFinish Nothing -> "Action finished" - MsgLauncherFinish (Just (ProcessDidNotStart name _e)) -> - "Could not start "+|name|+"" - MsgLauncherFinish (Just (ProcessHasExited name code)) -> - "Child process "+|name|+" exited with "+|statusText code|+"" + MsgLauncherFinish (Just exited) -> build $ toText exited MsgLauncherCleanup -> "Begin process cleanup" MsgLauncherCleanupTimedOut t -> diff --git a/lib/shelley/bench/Restore.hs b/lib/shelley/bench/Restore.hs index 73092f7d4a3..ec7cc392ed9 100644 --- a/lib/shelley/bench/Restore.hs +++ b/lib/shelley/bench/Restore.hs @@ -194,6 +194,8 @@ import Numeric ( showFFloat ) import Say ( sayErr ) +import System.Exit + ( exitWith ) import System.FilePath ( () ) import System.IO @@ -220,7 +222,7 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as T main :: IO () -main = execBenchWithNode argsNetworkConfig cardanoRestoreBench +main = execBenchWithNode argsNetworkConfig cardanoRestoreBench >>= exitWith {------------------------------------------------------------------------------- Shelley benchmarks From 3f5970478fa785132b26d19c21de0c5884483c1d Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Thu, 18 Mar 2021 17:34:52 +1000 Subject: [PATCH 3/4] buildkite nightly: Bump nixpkgs channel to 20.09 --- .buildkite/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/nightly.yml b/.buildkite/nightly.yml index 59c26798911..9989fc5f244 100644 --- a/.buildkite/nightly.yml +++ b/.buildkite/nightly.yml @@ -1,8 +1,8 @@ env: - NIX_PATH: "channel:nixos-19.03" BUILD_DIR: "/build/cardano-wallet" STACK_ROOT: "/build/cardano-wallet.stack" CACHE_DIR: "/cache/cardano-wallet" + NIX_PATH: "channel:nixos-20.09" SCRATCH_DIR: "/scratch/cardano-wallet" steps: From 3fed76f2b51b7ec3b450e781c744ce507560314a Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Thu, 18 Mar 2021 17:49:08 +1000 Subject: [PATCH 4/4] Nightly restore bench: fix temporary/cache paths Put TMPDIR in /scratch/tmp so that there's always enough space. Ensure that the node database path is set. It can also be cached between runs. --- .buildkite/bench-restore.sh | 15 ++++++++------- .buildkite/nightly.yml | 3 --- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.buildkite/bench-restore.sh b/.buildkite/bench-restore.sh index fb3e82c5c55..f0b5818239d 100755 --- a/.buildkite/bench-restore.sh +++ b/.buildkite/bench-restore.sh @@ -14,21 +14,22 @@ log=restore.log results=restore-$network.txt total_time=restore-time.txt -: "${NODE_DB:=$HOME/node-db-$network}" +if [ -n "${SCRATCH_DIR:-}" ]; then + mkdir -pv "$SCRATCH_DIR" + export TMPDIR="$SCRATCH_DIR/tmp" + mkdir -pv "$TMPDIR" +fi + +: "${node_db:=$HOME/node-db-$network}" echo "--- Build" nix-build -A benchmarks.cardano-wallet.restore -o bench-restore -bench="./bench-restore/bin/restore $network" +bench="./bench-restore/bin/restore $network --node-db $node_db" echo "--- Run benchmarks - $network" -if [ -n "${SCRATCH_DIR:-}" ]; then - mkdir -p "$SCRATCH_DIR" - export HOME="$SCRATCH_DIR" -fi - command time -o $total_time -v $bench +RTS -N2 -qg -A1m -I0 -T -M8G -h -RTS 2>&1 | tee $log grep -v INFO $log | awk '/All results/,EOF { print $0 }' > $results diff --git a/.buildkite/nightly.yml b/.buildkite/nightly.yml index 9989fc5f244..9dfef6817b6 100644 --- a/.buildkite/nightly.yml +++ b/.buildkite/nightly.yml @@ -1,7 +1,4 @@ env: - BUILD_DIR: "/build/cardano-wallet" - STACK_ROOT: "/build/cardano-wallet.stack" - CACHE_DIR: "/cache/cardano-wallet" NIX_PATH: "channel:nixos-20.09" SCRATCH_DIR: "/scratch/cardano-wallet"