Merge #2322

2322: [ADP-454] Runtime metrics r=hasufell a=hasufell ADP-454 <details> <summary>Teaser Prometheus</summary> ``` $ curl http://localhost:13798/metrics cardano_wallet_metrics_Stat_rtpriority_int 0 cardano_wallet_metrics_Stat_itrealvalue_int 0 rts_gc_par_max_bytes_copied 0 cardano_wallet_metrics_IO_syscr_int 54 cardano_wallet_metrics_Stat_minflt_int 1441 cardano_wallet_metrics_RTS_maxCompactBytes_B 0 cardano_wallet_metrics_Stat_cminflt_int 0 cardano_wallet_metrics_RTS_gcCopiedBytes_B 47608 rts_gc_num_gcs 221 cardano_wallet_metrics_Stat_enddata_int 29771744 cardano_wallet_metrics_Net_IpExt:OutOctets_B 1377542865 cardano_wallet_metrics_Stat_priority_int 20 cardano_wallet_metrics_IO_wchar_B 83 cardano_wallet_metrics_Stat_rsslim_int -1 cardano_wallet_metrics_RTS_cpuNs_ns 1920962 cardano_wallet_metrics_Stat_envend_int 140726935355228 cardano_wallet_metrics_Stat_pid_int 12809 cardano_wallet_metrics_Stat_ttynr_int 34823 cardano_wallet_metrics_Stat_startcode_int 4194304 rts_gc_mutator_wall_ms 9575 cardano_wallet_metrics_Stat_sigignore_int 0 rts_gc_gc_cpu_ms 105 cardano_wallet_metrics_Stat_cputicks_int 0 rts_gc_par_avg_bytes_copied 0 cardano_wallet_metrics_Stat_sigcatch_int 528390 cardano_wallet_metrics_Stat_majflt_int 0 cardano_wallet_metrics_Stat_cnswap_int 0 cardano_wallet_metrics_Stat_utime_int 0 cardano_wallet_metrics_Stat_blocked_int 0 cardano_wallet_metrics_Stat_cmajflt_int 0 rts_gc_init_cpu_ms 0 cardano_wallet_metrics_Stat_wchan_int 0 cardano_wallet_metrics_Stat_argend_int 140726935350409 cardano_wallet_metrics_IO_wbytes_B 0 cardano_wallet_metrics_Stat_startbrk_int 34394112 cardano_wallet_metrics_Stat_kstkeip_int 0 cardano_wallet_metrics_Stat_numthreads_int 6 cardano_wallet_metrics_Stat_session_int 26622 cardano_wallet_metrics_Stat_tpgid_int 12809 cardano_wallet_metrics_Mem_shared_int 5451 cardano_wallet_metrics_RTS_elapsedNs_ns 301678938 rts_gc_bytes_allocated 230364424 rts_gc_num_bytes_usage_samples 15 cardano_wallet_metrics_Mem_resident_int 6365 rts_gc_current_bytes_slop 227408 cardano_wallet_metrics_Stat_kstkesp_int 0 cardano_wallet_metrics_RTS_maxUsedMemBytes_B 2097152 cardano_wallet_metrics_Stat_argstart_int 140726935349989 cardano_wallet_metrics_Sys_Platform_int 1 rts_gc_max_bytes_slop 313624 cardano_wallet_metrics_IO_rchar_B 11062 rts_gc_par_tot_bytes_copied 0 cardano_wallet_metrics_Stat_flags_int 1077952512 cardano_wallet_metrics_Mem_data_int 11577 cardano_wallet_metrics_Stat_endcode_int 27238048 cardano_wallet_metrics_Mem_text_int 5626 cardano_wallet_metrics_RTS_gcElapsedNs_ns 82336 rts_gc_cumulative_bytes_used 68159136 cardano_wallet_metrics_Stat_rss_int 6365 cardano_wallet_metrics_Mem_size_int 268523351 ekg_server_timestamp_ms 1605554789360 cardano_wallet_metrics_Stat_cstime_int 0 cardano_wallet_metrics_RTS_maxLiveBytes_B 0 cardano_wallet_metrics_Stat_exitcode_int 0 cardano_wallet_metrics_Stat_guesttime_int 0 cardano_wallet_metrics_RTS_gcLiveBytes_B 133704 cardano_wallet_metrics_Stat_ppid_int 26622 rts_gc_gc_wall_ms 105 cardano_wallet_metrics_Stat_envstart_int 140726935350409 cardano_wallet_metrics_Stat_pgrp_int 12809 cardano_wallet_metrics_RTS_gcNum_int 1 rts_gc_mutator_cpu_ms 160 cardano_wallet_metrics_Stat_starttime_int 38375463 rts_gc_peak_megabytes_allocated 17 cardano_wallet_metrics_Stat_policy_int 0 cardano_wallet_metrics_Stat_startdata_int 27245632 cardano_wallet_metrics_IO_cxwbytes_B 0 cardano_wallet_metrics_Stat_vsize_int 1099871645696 cardano_wallet_metrics_RTS_maxSlopBytes_B 0 rts_gc_init_wall_ms 0 cardano_wallet_metrics_Stat_cutime_int 0 cardano_wallet_metrics_Net_IpExt:InOctets_B 12129589852 rts_gc_max_bytes_used 7362280 cardano_wallet_metrics_Stat_nswap_int 0 rts_gc_bytes_copied 27291728 cardano_wallet_metrics_Stat_blkio_int 0 cardano_wallet_metrics_IO_rbytes_B 0 cardano_wallet_metrics_Stat_startstack_int 140726935348176 cardano_wallet_metrics_RTS_maxLargeBytes_B 0 cardano_wallet_metrics_Stat_cguesttime_int 0 cardano_wallet_metrics_RTS_gcMajorNum_int 0 cardano_wallet_metrics_Sys_Pid_int 12809 rts_gc_wall_ms 9681 cardano_wallet_metrics_RTS_bytesAllocated_B 1129192 rts_gc_current_bytes_used 5765040 cardano_wallet_metrics_RTS_gcCpuNs_ns 82551 cardano_wallet_metrics_Stat_stime_int 0 cardano_wallet_metrics_Stat_signal_int 0 rts_gc_cpu_ms 266 cardano_wallet_metrics_Stat_exitsignal_int 17 cardano_wallet_metrics_Stat_processor_int 5 cardano_wallet_metrics_RTS_liveBytes_B 0 cardano_wallet_metrics_Stat_nice_int 0 cardano_wallet_metrics_IO_syscw_int 11 ``` </details> <details> <summary>Teaser EKG</summary> ``` curl -H "Accept: application/json" http://localhost:13788/ | jq . ``` ```json { "iohk-monitoring version": { "type": "l", "val": "0.1.10.1" }, "ekg": { "server_timestamp_ms": { "type": "c", "val": 1605553522731 } }, "rts": { "gc": { "gc_cpu_ms": { "type": "c", "val": 124 }, "mutator_wall_ms": { "type": "c", "val": 12178 }, "mutator_cpu_ms": { "type": "c", "val": 598 }, "gc_wall_ms": { "type": "c", "val": 118 }, "wall_ms": { "type": "c", "val": 12296 }, "bytes_copied": { "type": "c", "val": 60092216 }, "init_wall_ms": { "type": "c", "val": 0 }, "init_cpu_ms": { "type": "c", "val": 0 }, "max_bytes_used": { "type": "g", "val": 26806536 }, "max_bytes_slop": { "type": "g", "val": 1001208 }, "num_bytes_usage_samples": { "type": "c", "val": 12 }, "peak_megabytes_allocated": { "type": "g", "val": 40 }, "cpu_ms": { "type": "c", "val": 723 }, "current_bytes_used": { "type": "g", "val": 23334656 }, "bytes_allocated": { "type": "c", "val": 529295488 }, "par_max_bytes_copied": { "type": "g", "val": 0 }, "current_bytes_slop": { "type": "g", "val": 758016 }, "cumulative_bytes_used": { "type": "c", "val": 122968880 }, "num_gcs": { "type": "c", "val": 503 }, "par_tot_bytes_copied": { "type": "g", "val": 0 }, "par_avg_bytes_copied": { "type": "g", "val": 0 } } }, "cardano-wallet": { "wallet-engine": { "type": "l", "val": "\"\\\"9914f38a: local tip: 46f10b60<-[b40aa5a2-11184306#2045091]\\\"\"" }, "pools-engine": { "type": "l", "val": "\"\\\"Applying blocks [11184306 ... 11184306]\\\"\"" }, "metrics": { "Net": { "IpExt:InOctets": { "B": { "type": "g", "val": 12110694532 } }, "IpExt:OutOctets": { "B": { "type": "g", "val": 1374876709 } } }, "Mem": { "size": { "int": { "type": "g", "val": 268523646 } }, "text": { "int": { "type": "g", "val": 5901 } }, "data": { "int": { "type": "g", "val": 12365 } }, "resident": { "int": { "type": "g", "val": 7752 } }, "shared": { "int": { "type": "g", "val": 6347 } } }, "IO": { "rchar": { "B": { "type": "g", "val": 11062 } }, "syscw": { "int": { "type": "g", "val": 21 } }, "rbytes": { "B": { "type": "g", "val": 0 } }, "wchar": { "B": { "type": "g", "val": 93 } }, "cxwbytes": { "B": { "type": "g", "val": 0 } }, "syscr": { "int": { "type": "g", "val": 54 } }, "wbytes": { "B": { "type": "g", "val": 0 } } }, "Sys": { "Platform": { "int": { "type": "g", "val": 1 } }, "Pid": { "int": { "type": "g", "val": 7172 } } }, "Stat": { "minflt": { "int": { "type": "g", "val": 2009 } }, "rtpriority": { "int": { "type": "g", "val": 0 } }, "flags": { "int": { "type": "g", "val": 1077952512 } }, "endcode": { "int": { "type": "g", "val": 28361072 } }, "envend": { "int": { "type": "g", "val": 140736416722780 } }, "rsslim": { "int": { "type": "g", "val": -1 } }, "rss": { "int": { "type": "g", "val": 7752 } }, "priority": { "int": { "type": "g", "val": 20 } }, "guesttime": { "int": { "type": "g", "val": 0 } }, "starttime": { "int": { "type": "g", "val": 38248539 } }, "cnswap": { "int": { "type": "g", "val": 0 } }, "sigcatch": { "int": { "type": "g", "val": 544774 } }, "startdata": { "int": { "type": "g", "val": 28367904 } }, "majflt": { "int": { "type": "g", "val": 0 } }, "cputicks": { "int": { "type": "g", "val": 2 } }, "numthreads": { "int": { "type": "g", "val": 6 } }, "blkio": { "int": { "type": "g", "val": 0 } }, "nswap": { "int": { "type": "g", "val": 0 } }, "vsize": { "int": { "type": "g", "val": 1099872854016 } }, "argend": { "int": { "type": "g", "val": 140736416717961 } }, "startbrk": { "int": { "type": "g", "val": 53985280 } }, "stime": { "int": { "type": "g", "val": 0 } }, "processor": { "int": { "type": "g", "val": 7 } }, "argstart": { "int": { "type": "g", "val": 140736416717541 } }, "itrealvalue": { "int": { "type": "g", "val": 0 } }, "cminflt": { "int": { "type": "g", "val": 0 } }, "enddata": { "int": { "type": "g", "val": 30976544 } }, "ppid": { "int": { "type": "g", "val": 26622 } }, "sigignore": { "int": { "type": "g", "val": 0 } }, "startcode": { "int": { "type": "g", "val": 4194304 } }, "envstart": { "int": { "type": "g", "val": 140736416717961 } }, "cstime": { "int": { "type": "g", "val": 0 } }, "pgrp": { "int": { "type": "g", "val": 7172 } }, "exitcode": { "int": { "type": "g", "val": 0 } }, "pid": { "int": { "type": "g", "val": 7172 } }, "ttynr": { "int": { "type": "g", "val": 34823 } }, "policy": { "int": { "type": "g", "val": 0 } }, "utime": { "int": { "type": "g", "val": 2 } }, "blocked": { "int": { "type": "g", "val": 0 } }, "cmajflt": { "int": { "type": "g", "val": 0 } }, "cutime": { "int": { "type": "g", "val": 0 } }, "wchan": { "int": { "type": "g", "val": 0 } }, "kstkeip": { "int": { "type": "g", "val": 0 } }, "startstack": { "int": { "type": "g", "val": 140736416715888 } }, "session": { "int": { "type": "g", "val": 26622 } }, "exitsignal": { "int": { "type": "g", "val": 17 } }, "tpgid": { "int": { "type": "g", "val": 7172 } }, "cguesttime": { "int": { "type": "g", "val": 0 } }, "signal": { "int": { "type": "g", "val": 0 } }, "kstkesp": { "int": { "type": "g", "val": 0 } }, "nice": { "int": { "type": "g", "val": 0 } } }, "RTS": { "maxUsedMemBytes": { "B": { "type": "g", "val": 2097152 } }, "gcCpuNs": { "ns": { "type": "g", "val": 128700 } }, "maxCompactBytes": { "B": { "type": "g", "val": 0 } }, "maxLargeBytes": { "B": { "type": "g", "val": 0 } }, "bytesAllocated": { "B": { "type": "g", "val": 1129192 } }, "cpuNs": { "ns": { "type": "g", "val": 3184072 } }, "gcMajorNum": { "int": { "type": "g", "val": 0 } }, "gcCopiedBytes": { "B": { "type": "g", "val": 47608 } }, "elapsedNs": { "ns": { "type": "g", "val": 302668800 } }, "liveBytes": { "B": { "type": "g", "val": 0 } }, "gcNum": { "int": { "type": "g", "val": 1 } }, "maxSlopBytes": { "B": { "type": "g", "val": 0 } }, "gcElapsedNs": { "ns": { "type": "g", "val": 128435 } }, "maxLiveBytes": { "B": { "type": "g", "val": 0 } }, "gcLiveBytes": { "B": { "type": "g", "val": 133704 } } } }, "application": { "type": "l", "val": "\"\\\"Node is Haskell Node on testnet (1097911063).\\\"\"" }, "main": { "type": "l", "val": "\"\\\"Wallet backend server listening on 127.0.0.1:8090\\\"\"" }, "ntp-client": { "type": "l", "val": "\"\\\"local clock is drifting by -3214\\206\\188s\\\"\"" }, "network": { "type": "l", "val": "\"\\\"Query GetUpdateInterfaceState took 0.001308712s\\\"\"" }, "pools-db": { "type": "l", "val": "\"\\\"Will open db at ../cardano-node/run/wallet-state-test/stake-pools.sqlite\\\"\"" }, "wallet-db": { "type": "l", "val": "\"\\\"rnd_state_address table does not contain required field 'account_ix'. Adding this field with a default value of 2147483648.\\\"\"" } } } ``` </details> Co-authored-by: Julian Ospald <[email protected]>
cardano-foundation · Dec 2, 2020 · 4f7300f · 4f7300f
2 parents ec22c1b + 4e6af1a
commit 4f7300f
Show file tree

Hide file tree

Showing 9 changed files with 162 additions and 28 deletions.
diff --git a/cabal.project b/cabal.project
@@ -1,4 +1,5 @@
 -- Generated by stackage-to-hackage
+--
 
 index-state: 2020-11-11T15:40:10Z
 

diff --git a/lib/cli/cardano-wallet-cli.cabal b/lib/cli/cardano-wallet-cli.cabal
@@ -32,6 +32,7 @@ library
       aeson
     , aeson-pretty
     , ansi-terminal
+    , async
     , base
     , bytestring
     , cardano-addresses

diff --git a/lib/cli/src/Cardano/CLI.hs b/lib/cli/src/Cardano/CLI.hs
@@ -11,6 +11,7 @@
 {-# LANGUAGE GeneralizedNewtypeDeriving #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE MultiParamTypeClasses #-}
+{-# LANGUAGE NumericUnderscores #-}
 {-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
@@ -94,6 +95,9 @@ module Cardano.CLI
     , getDataDir
     , setupDirectory
     , waitForService
+    , getPrometheusURL
+    , getEKGURL
+    , ekgEnabled
     , WaitForServiceLog (..)
     ) where
 
@@ -104,8 +108,14 @@ import Cardano.BM.Backend.Switchboard
     ( Switchboard )
 import Cardano.BM.Configuration.Static
     ( defaultConfigStdout )
+import Cardano.BM.Counters
+    ( readCounters )
+import Cardano.BM.Data.Configuration
+    ( Endpoint (..) )
+import Cardano.BM.Data.Counter
+    ( Counter (..), nameCounter )
 import Cardano.BM.Data.LogItem
-    ( LoggerName )
+    ( LOContent (..), LoggerName, PrivacyAnnotation (..), mkLOMeta )
 import Cardano.BM.Data.Output
     ( ScribeDefinition (..)
     , ScribeFormat (..)
@@ -115,12 +125,14 @@ import Cardano.BM.Data.Output
     )
 import Cardano.BM.Data.Severity
     ( Severity (..) )
+import Cardano.BM.Data.SubTrace
+    ( SubTrace (..) )
 import Cardano.BM.Data.Tracer
     ( HasPrivacyAnnotation (..), HasSeverityAnnotation (..) )
 import Cardano.BM.Setup
     ( setupTrace_, shutdown )
 import Cardano.BM.Trace
-    ( Trace, appendName, logDebug )
+    ( Trace, appendName, logDebug, traceNamedObject )
 import Cardano.Mnemonic
     ( MkSomeMnemonic (..), SomeMnemonic (..) )
 import Cardano.Wallet.Api.Client
@@ -185,10 +197,14 @@ import Control.Applicative
     ( optional, some, (<|>) )
 import Control.Arrow
     ( first, left )
+import Control.Concurrent
+    ( threadDelay )
 import Control.Exception
     ( bracket, catch )
 import Control.Monad
-    ( join, unless, void, when )
+    ( forM_, forever, join, unless, void, when )
+import Control.Monad.IO.Class
+    ( MonadIO )
 import Control.Tracer
     ( Tracer, traceWith )
 import Data.Aeson
@@ -200,7 +216,7 @@ import Data.Char
 import Data.List.NonEmpty
     ( NonEmpty (..) )
 import Data.Maybe
-    ( fromMaybe )
+    ( fromMaybe, isJust )
 import Data.Quantity
     ( Quantity (..) )
 import Data.String
@@ -286,6 +302,8 @@ import System.Directory
     , doesFileExist
     , getXdgDirectory
     )
+import System.Environment
+    ( lookupEnv )
 import System.Exit
     ( exitFailure, exitSuccess )
 import System.FilePath
@@ -309,8 +327,10 @@ import System.IO
 
 import qualified Cardano.BM.Configuration.Model as CM
 import qualified Cardano.BM.Data.BackendKind as CM
+import qualified Cardano.BM.Data.Observable as Obs
 import qualified Command.Key as Key
 import qualified Command.RecoveryPhrase as RecoveryPhrase
+import qualified Control.Concurrent.Async as Async
 import qualified Data.Aeson as Aeson
 import qualified Data.Aeson.Encode.Pretty as Aeson
 import qualified Data.Aeson.Types as Aeson
@@ -1576,26 +1596,85 @@ mkScribeId :: LogOutput -> ScribeId
 mkScribeId (LogToStdout _) = "StdoutSK::text"
 mkScribeId (LogToFile file _) = T.pack $ "FileSK::" <> file
 
+getPrometheusURL :: IO (Maybe (String, Port "Prometheus"))
+getPrometheusURL = do
+    prometheus_port <- lookupEnv "CARDANO_WALLET_PROMETHEUS_PORT"
+    prometheus_host <- fromMaybe "localhost" <$> lookupEnv "CARDANO_WALLET_PROMETHEUS_HOST"
+    case (prometheus_host, prometheus_port) of
+        (host, Just port) ->
+            case fromText @(Port "Prometheus") $ T.pack port of
+                Right port' -> pure $ Just (host, port')
+                _ -> do
+                    TIO.hPutStr stderr
+                        "Port value for prometheus metrics invalid. Will be disabled."
+                    pure Nothing
+        _ -> pure Nothing
+
+getEKGURL :: IO (Maybe (String, Port "EKG"))
+getEKGURL = do
+    ekg_port <- lookupEnv "CARDANO_WALLET_EKG_PORT"
+    ekg_host <- fromMaybe "localhost" <$> lookupEnv "CARDANO_WALLET_EKG_HOST"
+    case (ekg_host, ekg_port) of
+        (host, Just port) ->
+            case fromText @(Port "EKG") $ T.pack port of
+                Right port' -> pure $ Just (host, port')
+                _ -> do
+                    TIO.hPutStr stderr
+                        "Port value for EKB metrics invalid. Will be disabled."
+                    pure Nothing
+        _ -> pure Nothing
+
+ekgEnabled :: IO Bool
+ekgEnabled = isJust <$> getEKGURL
 
 -- | Initialize logging at the specified minimum 'Severity' level.
 initTracer
     :: LoggerName
     -> [LogOutput]
     -> IO (Switchboard Text, (CM.Configuration, Trace IO Text))
 initTracer loggerName outputs = do
+    prometheusHP <- getPrometheusURL
+    ekgHP <- getEKGURL
     cfg <- do
         c <- defaultConfigStdout
-        CM.setSetupBackends c [CM.KatipBK, CM.AggregationBK]
+        CM.setSetupBackends c [CM.KatipBK, CM.AggregationBK, CM.EKGViewBK, CM.EditorBK]
+        CM.setDefaultBackends c [CM.KatipBK, CM.EKGViewBK]
         CM.setSetupScribes c $ map mkScribe outputs
         CM.setDefaultScribes c $ map mkScribeId outputs
+        CM.setBackends c "test-cluster.metrics" (Just [CM.EKGViewBK])
+        CM.setBackends c "cardano-wallet.metrics" (Just [CM.EKGViewBK])
+        forM_ ekgHP $ \(h, p) -> do
+            CM.setEKGBindAddr c $ Just (Endpoint (h, getPort p))
+        forM_ prometheusHP $ \(h, p) ->
+            CM.setPrometheusBindAddr c $ Just (h, getPort p)
         pure c
     (tr, sb) <- setupTrace_ cfg loggerName
+    ekgEnabled >>= flip when (startCapturingMetrics tr)
     pure (sb, (cfg, tr))
+  where
+    -- https://github.com/input-output-hk/cardano-node/blob/f7d57e30c47028ba2aeb306a4f21b47bb41dec01/cardano-node/src/Cardano/Node/Configuration/Logging.hs#L224
+    startCapturingMetrics :: Trace IO Text -> IO ()
+    startCapturingMetrics trace0 = do
+      let trace = appendName "metrics" trace0
+          counters = [Obs.MemoryStats, Obs.ProcessStats
+            , Obs.NetStats, Obs.IOStats, Obs.GhcRtsStats, Obs.SysStats]
+      _ <- Async.async $ forever $ do
+        cts <- readCounters (ObservableTraceSelf counters)
+        traceCounters trace cts
+        threadDelay 30_000_000   -- 30 seconds
+      pure ()
+     where
+       traceCounters :: forall m a. MonadIO m => Trace m a -> [Counter] -> m ()
+       traceCounters _tr [] = return ()
+       traceCounters tr (c@(Counter _ct cn cv) : cs) = do
+         mle <- mkLOMeta Notice Confidential
+         traceNamedObject tr (mle, LogValue (nameCounter c <> "." <> cn) cv)
+         traceCounters tr cs
 
 -- | See 'withLoggingNamed'
 withLogging
     :: [LogOutput]
-    -> ((CM.Configuration, Trace IO Text) -> IO a)
+    -> ((Switchboard Text, (CM.Configuration, Trace IO Text)) -> IO a)
     -> IO a
 withLogging =
     withLoggingNamed "cardano-wallet"
@@ -1605,10 +1684,10 @@ withLogging =
 withLoggingNamed
     :: LoggerName
     -> [LogOutput]
-    -> ((CM.Configuration, Trace IO Text) -> IO a)
+    -> ((Switchboard Text, (CM.Configuration, Trace IO Text)) -> IO a)
     -- ^ The action to run with logging configured.
     -> IO a
-withLoggingNamed loggerName outputs action = bracket before after (action . snd)
+withLoggingNamed loggerName outputs = bracket before after
   where
     before = initTracer loggerName outputs
     after (sb, (_, tr)) = do

diff --git a/lib/shelley/cardano-wallet.cabal b/lib/shelley/cardano-wallet.cabal
@@ -118,6 +118,7 @@ executable cardano-wallet
     , cardano-wallet
     , contra-tracer
     , iohk-monitoring
+    , lobemo-backend-ekg
     , network
     , optparse-applicative
     , text
@@ -148,6 +149,7 @@ executable shelley-test-cluster
     , cardano-wallet
     , contra-tracer
     , iohk-monitoring
+    , lobemo-backend-ekg
     , text
     , text-class
   hs-source-dirs:
@@ -234,6 +236,7 @@ test-suite integration
     , hspec
     , http-client
     , iohk-monitoring
+    , lobemo-backend-ekg
     , text
     , text-class
   build-tools:

diff --git a/lib/shelley/exe/cardano-wallet.hs b/lib/shelley/exe/cardano-wallet.hs
@@ -27,6 +27,8 @@ import Prelude
 
 import Cardano.BM.Data.Severity
     ( Severity (..) )
+import Cardano.BM.Plugin
+    ( loadPlugin )
 import Cardano.BM.Trace
     ( Trace, appendName, logDebug, logError, logInfo, logNotice )
 import Cardano.CLI
@@ -43,6 +45,7 @@ import Cardano.CLI
     , cmdWallet
     , cmdWalletCreate
     , databaseOption
+    , ekgEnabled
     , enableWindowsANSI
     , helperTracing
     , hostPreferenceOption
@@ -102,7 +105,7 @@ import Cardano.Wallet.Version
 import Control.Applicative
     ( Const (..), optional )
 import Control.Monad
-    ( void )
+    ( void, when )
 import Control.Monad.Trans.Except
     ( runExceptT )
 import Control.Tracer
@@ -134,6 +137,7 @@ import System.Environment
 import System.Exit
     ( ExitCode (..), exitWith )
 
+import qualified Cardano.BM.Backend.EKGView as EKG
 import qualified Cardano.Wallet.Version as V
 import qualified Data.Text as T
 
@@ -294,7 +298,8 @@ withTracers
     -> (Trace IO MainLog -> Tracers IO -> IO a)
     -> IO a
 withTracers logOpt action =
-    withLogging [LogToStdout (loggingMinSeverity logOpt)] $ \(_, tr) -> do
+    withLogging [LogToStdout (loggingMinSeverity logOpt)] $ \(sb, (cfg, tr)) -> do
+        ekgEnabled >>= flip when (EKG.plugin cfg tr sb >>= loadPlugin sb)
         let trMain = appendName "main" (transformTextTrace tr)
         let tracers = setupTracers (loggingTracers logOpt) tr
         logInfo trMain $ MsgVersion V.version gitRevision

diff --git a/lib/shelley/exe/shelley-test-cluster.hs b/lib/shelley/exe/shelley-test-cluster.hs
@@ -14,8 +14,16 @@ import Cardano.BM.Data.Severity
     ( Severity (..) )
 import Cardano.BM.Data.Tracer
     ( HasPrivacyAnnotation (..), HasSeverityAnnotation (..) )
+import Cardano.BM.Plugin
+    ( loadPlugin )
 import Cardano.CLI
-    ( LogOutput (..), withLoggingNamed )
+    ( LogOutput (..)
+    , Port
+    , ekgEnabled
+    , getEKGURL
+    , getPrometheusURL
+    , withLoggingNamed
+    )
 import Cardano.Startup
     ( setDefaultFilePermissions, withUtf8Encoding )
 import Cardano.Wallet.Api.Types
@@ -54,7 +62,7 @@ import Cardano.Wallet.Shelley.Launch
 import Control.Arrow
     ( first )
 import Control.Monad
-    ( void )
+    ( void, when )
 import Control.Tracer
     ( contramap, traceWith )
 import Data.Proxy
@@ -68,6 +76,7 @@ import System.IO
 import Test.Integration.Faucet
     ( genRewardAccounts, mirMnemonics, shelleyIntegrationTestFunds )
 
+import qualified Cardano.BM.Backend.EKGView as EKG
 import qualified Data.Text as T
 
 -- |
@@ -208,8 +217,8 @@ main = withUtf8Encoding $ do
 
     poolConfigs <- poolConfigsFromEnv
     withLoggingNamed "cardano-wallet" walletLogs
-        $ \(_, trWallet) -> withLoggingNamed "test-cluster" clusterLogs
-        $ \(_, trCluster) -> withSystemTempDir (trMessageText trCluster) "testCluster"
+        $ \(sb, (cfg, trWallet)) -> withLoggingNamed "test-cluster" clusterLogs
+        $ \(_, (_, trCluster)) -> withSystemTempDir (trMessageText trCluster) "testCluster"
         $ \dir -> withTempDir (trMessageText trCluster) dir "wallets"
         $ \db -> withCluster
             (contramap MsgCluster $ trMessageText trCluster)
@@ -219,7 +228,7 @@ main = withUtf8Encoding $ do
             Nothing
             whenByron
             (whenShelley dir (trMessageText trCluster))
-            (whenReady trWallet (trMessageText trCluster) db)
+            (whenReady sb cfg trWallet (trMessageText trCluster) db)
   where
     whenByron _ = pure ()
 
@@ -233,9 +242,19 @@ main = withUtf8Encoding $ do
         sendFaucetFundsTo trCluster' dir addresses
         moveInstantaneousRewardsTo trCluster' dir rewards
 
-    whenReady tr trCluster db (RunningNode socketPath block0 (gp, vData)) = do
+    whenReady sb cfg tr trCluster db (RunningNode socketPath block0 (gp, vData)) = do
+        ekgEnabled >>= flip when (EKG.plugin cfg tr sb >>= loadPlugin sb)
+
         let tracers = setupTracers (tracerSeverities (Just Info)) tr
         listen <- walletListenFromEnv
+        prometheusUrl <- (maybe "none"
+                (\(h, p) -> T.pack h <> ":" <> toText @(Port "Prometheus") p)
+            )
+            <$> getPrometheusURL
+        ekgUrl <- (maybe "none"
+                (\(h, p) -> T.pack h <> ":" <> toText @(Port "EKG") p)
+            )
+            <$> getEKGURL
         void $ serveWallet @(IO Shelley)
             (SomeNetworkDiscriminant $ Proxy @'Mainnet)
             tracers
@@ -249,26 +268,30 @@ main = withUtf8Encoding $ do
             socketPath
             block0
             (gp, vData)
-            (traceWith trCluster . MsgBaseUrl . T.pack . show)
+            (\u -> traceWith trCluster $ MsgBaseUrl (T.pack . show $ u)
+                ekgUrl prometheusUrl)
 
 -- Logging
 
 data TestsLog
-    = MsgBaseUrl Text
+    = MsgBaseUrl Text Text Text -- wallet url, ekg url, prometheus url
     | MsgSettingUpFaucet
     | MsgCluster ClusterLog
     deriving (Show)
 
 instance ToText TestsLog where
     toText = \case
-        MsgBaseUrl addr ->
-            "Wallet backend server listening on " <> T.pack (show addr)
+        MsgBaseUrl walletUrl ekgUrl prometheusUrl -> mconcat
+            [ "Wallet url: " , walletUrl
+            , ", EKG url: " , ekgUrl
+            , ", Prometheus url:", prometheusUrl
+            ]
         MsgSettingUpFaucet -> "Setting up faucet..."
         MsgCluster msg -> toText msg
 
 instance HasPrivacyAnnotation TestsLog
 instance HasSeverityAnnotation TestsLog where
     getSeverityAnnotation = \case
         MsgSettingUpFaucet -> Notice
-        MsgBaseUrl _ -> Notice
+        MsgBaseUrl {} -> Notice
         MsgCluster msg -> getSeverityAnnotation msg