Skip to content

Commit

Permalink
cardano-node: implement --shutdown-on-block-synced
Browse files Browse the repository at this point in the history
  • Loading branch information
deepfire committed Jun 1, 2022
1 parent 982327a commit 5660b57
Show file tree
Hide file tree
Showing 8 changed files with 136 additions and 80 deletions.
107 changes: 78 additions & 29 deletions cardano-node/src/Cardano/Node/Handlers/Shutdown.hs
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE PackageImports #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TupleSections #-}
{-# OPTIONS_GHC -Wno-orphans #-}

module Cardano.Node.Handlers.Shutdown
(
( SlotOrBlock (..)
, parseShutdownOnLimit

-- * Generalised shutdown handling
ShutdownConfig (..)
, ShutdownConfig (..)
, withShutdownHandling

, ShutdownTrace (..)
Expand All @@ -20,27 +27,50 @@ module Cardano.Node.Handlers.Shutdown
where

import Data.Aeson (FromJSON, ToJSON)
import Generic.Data (Generic)
import Generic.Data.Orphans ()
import Prelude
import Cardano.Prelude

import Control.Concurrent.Async (race_)
import Control.Exception
import Control.Monad
import Data.Text (Text, pack)
import Data.Text (pack)
import qualified GHC.IO.Handle.FD as IO (fdToHandle)
import System.Exit
import qualified Options.Applicative as Opt
import qualified System.IO as IO
import qualified System.IO.Error as IO
import System.Posix.Types (Fd (Fd))

import Cardano.Slotting.Slot (WithOrigin (..))
import "contra-tracer" Control.Tracer
import Ouroboros.Consensus.Block (Header)
import qualified Ouroboros.Consensus.Storage.ChainDB as ChainDB
import Ouroboros.Consensus.Util.ResourceRegistry (ResourceRegistry)
import Ouroboros.Consensus.Util.STM (Watcher (..), forkLinkedWatcher)
import Ouroboros.Network.Block (MaxSlotNo (..), SlotNo, pointSlot)
import Ouroboros.Network.Block (HasHeader, BlockNo (..), SlotNo (..), pointSlot)


data SlotOrBlock f
= ASlot !(f SlotNo)
| ABlock !(f BlockNo)
deriving (Generic)

deriving instance Eq (SlotOrBlock Identity)
deriving instance Show (SlotOrBlock Identity)
deriving instance FromJSON (SlotOrBlock Identity)
deriving instance ToJSON (SlotOrBlock Identity)

parseShutdownOnLimit :: Opt.Parser (Maybe (SlotOrBlock Identity))
parseShutdownOnLimit =
optional (Opt.option (ASlot . Identity . SlotNo <$> Opt.auto) (
Opt.long "shutdown-on-slot-synced"
<> Opt.metavar "SLOT"
<> Opt.help "Shut down the process after ChainDB is synced up to the specified slot"
<> Opt.hidden
))
<|>
optional (Opt.option (ABlock . Identity . BlockNo <$> Opt.auto) (
Opt.long "shutdown-on-block-synced"
<> Opt.metavar "BLOCK"
<> Opt.help "Shut down the process after ChainDB is synced up to the specified block"
<> Opt.hidden
))

data ShutdownTrace
= ShutdownRequested
Expand All @@ -51,14 +81,21 @@ data ShutdownTrace
-- ^ Received shutdown request but found unexpected input in --shutdown-ipc FD:
| RequestingShutdown Text
-- ^ Ringing the node shutdown doorbell for reason
| ShutdownArmedAtSlot SlotNo
-- ^ Will terminate upon reaching maxSlot
| ShutdownArmedAt (SlotOrBlock Identity)
-- ^ Will terminate upon reaching a ChainDB sync limit
deriving (Generic, FromJSON, ToJSON)

deriving instance FromJSON BlockNo
deriving instance ToJSON BlockNo

newtype AndWithOrigin a = AndWithOrigin (a, WithOrigin a) deriving (Eq)

deriving instance Eq (SlotOrBlock AndWithOrigin)

data ShutdownConfig
= ShutdownConfig
{ scIPC :: !(Maybe Fd)
, scOnSlotSynced :: !(Maybe MaxSlotNo)
{ scIPC :: !(Maybe Fd)
, scOnSyncLimit :: !(Maybe (SlotOrBlock Identity))
}
deriving (Eq, Show)

Expand Down Expand Up @@ -92,28 +129,40 @@ withShutdownHandling ShutdownConfig{scIPC = Just fd} tr action = do
-- | Spawn a thread that would cause node to shutdown upon ChainDB reaching the
-- configuration-defined slot.
maybeSpawnOnSlotSyncedShutdownHandler
:: ShutdownConfig
:: HasHeader (Header blk)
=> ShutdownConfig
-> Tracer IO ShutdownTrace
-> ResourceRegistry IO
-> ChainDB.ChainDB IO blk
-> IO ()
maybeSpawnOnSlotSyncedShutdownHandler sc tr registry chaindb =
case scOnSlotSynced sc of
Just (MaxSlotNo maxSlot) -> do
traceWith tr (ShutdownArmedAtSlot maxSlot)
spawnSlotLimitTerminator maxSlot
_ -> pure ()
case scOnSyncLimit sc of
Nothing -> pure ()
Just lim -> do
traceWith tr (ShutdownArmedAt lim)
spawnLimitTerminator lim
where
spawnSlotLimitTerminator :: SlotNo -> IO ()
spawnSlotLimitTerminator maxSlot =
spawnLimitTerminator :: SlotOrBlock Identity -> IO ()
spawnLimitTerminator limit =
void $ forkLinkedWatcher registry "slotLimitTerminator" Watcher {
wFingerprint = id
wFingerprint = identity
, wInitial = Nothing
, wReader =
case limit of
ASlot (Identity x) -> ASlot . AndWithOrigin . (x,) <$>
(pointSlot <$> ChainDB.getTipPoint chaindb)
ABlock (Identity x) -> ABlock . AndWithOrigin . (x,) <$>
ChainDB.getTipBlockNo chaindb
, wNotify = \case
Origin -> pure ()
At cur -> when (cur >= maxSlot) $ do
traceWith tr (RequestingShutdown $ "spawnSlotLimitTerminator: reached target "
<> (pack . show) cur)
throwIO ExitSuccess
, wReader = pointSlot <$> ChainDB.getTipPoint chaindb
ASlot (AndWithOrigin (lim, At cur)) ->
when (cur >= lim) $ do
traceWith tr (RequestingShutdown $ "spawnLimitTerminator: reached target slot "
<> (pack . show) cur)
throwIO ExitSuccess
ABlock (AndWithOrigin (lim, At cur)) ->
when (cur >= lim) $ do
traceWith tr (RequestingShutdown $ "spawnLimitTerminator: reached target block "
<> (pack . show) cur)
throwIO ExitSuccess
_ -> pure ()
}
16 changes: 2 additions & 14 deletions cardano-node/src/Cardano/Node/Parsers.hs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ import qualified Options.Applicative as Opt
import qualified Options.Applicative.Help as OptI
import System.Posix.Types (Fd (..))

import Ouroboros.Network.Block (MaxSlotNo (..), SlotNo (..))

import Ouroboros.Consensus.Mempool.API (MempoolCapacityBytes (..),
MempoolCapacityBytesOverride (..))
import Ouroboros.Consensus.Storage.LedgerDB.DiskPolicy (SnapshotInterval (..))
Expand Down Expand Up @@ -66,7 +64,7 @@ nodeRunParser = do

validate <- lastOption parseValidateDB
shutdownIPC <- lastOption parseShutdownIPC
shutdownOnSlot <- lastOption parseShutdownOnSlotSynced
shutdownOnLimit <- lastOption parseShutdownOnLimit

maybeMempoolCapacityOverride <- lastOption parseMempoolCapacityOverride

Expand All @@ -93,7 +91,7 @@ nodeRunParser = do
}
, pncValidateDB = validate
, pncShutdownConfig =
Last . Just $ ShutdownConfig (getLast shutdownIPC) (getLast shutdownOnSlot)
Last . Just $ ShutdownConfig (getLast shutdownIPC) (join $ getLast shutdownOnLimit)
, pncProtocolConfig = mempty
, pncMaxConcurrencyBulkSync = mempty
, pncMaxConcurrencyDeadline = mempty
Expand Down Expand Up @@ -215,16 +213,6 @@ parseShutdownIPC =
<> hidden
)

parseShutdownOnSlotSynced :: Parser MaxSlotNo
parseShutdownOnSlotSynced =
fmap (fromMaybe NoMaxSlotNo) $
optional $ option (MaxSlotNo . SlotNo <$> auto) (
long "shutdown-on-slot-synced"
<> metavar "SLOT"
<> help "Shut down the process after ChainDB is synced up to the specified slot"
<> hidden
)

parseTopologyFile :: Parser FilePath
parseTopologyFile =
strOption (
Expand Down
22 changes: 11 additions & 11 deletions cardano-node/src/Cardano/Node/Tracing/Tracers/Shutdown.hs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ module Cardano.Node.Tracing.Tracers.Shutdown

import Cardano.Logging
import Cardano.Node.Handlers.Shutdown
import Data.Aeson (ToJSON (..), Value (..), (.=))
import Data.Aeson (Value (..), (.=))
import Data.Monoid (mconcat, (<>))
import Data.Text (Text, pack)
import Prelude (show)
Expand All @@ -29,15 +29,15 @@ namesForShutdown = \case
AbnormalShutdown{} -> ["AbnormalShutdown"]
ShutdownUnexpectedInput{} -> ["ShutdownUnexpectedInput"]
RequestingShutdown{} -> ["RequestingShutdown"]
ShutdownArmedAtSlot{} -> ["ShutdownArmedAtSlot"]
ShutdownArmedAt{} -> ["ShutdownArmedAt"]

severityShutdown :: ShutdownTrace -> SeverityS
severityShutdown = \case
ShutdownRequested{} -> Warning
AbnormalShutdown{} -> Error
ShutdownUnexpectedInput{} -> Error
RequestingShutdown{} -> Warning
ShutdownArmedAtSlot{} -> Warning
ShutdownArmedAt{} -> Warning

ppShutdownTrace :: ShutdownTrace -> Text
ppShutdownTrace = \case
Expand All @@ -46,7 +46,7 @@ ppShutdownTrace = \case
ShutdownUnexpectedInput text ->
"Received shutdown request but found unexpected input in --shutdown-ipc FD: " <> text
RequestingShutdown reason -> "Ringing the node shutdown doorbell: " <> reason
ShutdownArmedAtSlot slot -> "Will terminate upon reaching " <> pack (show slot)
ShutdownArmedAt lim -> "Will terminate upon reaching " <> pack (show lim)

instance LogFormatting ShutdownTrace where
forHuman = ppShutdownTrace
Expand All @@ -58,13 +58,13 @@ instance LogFormatting ShutdownTrace where
mconcat [ "kind" .= String "AbnormalShutdown" ]
ShutdownUnexpectedInput text ->
mconcat [ "kind" .= String "AbnormalShutdown"
, "unexpected" .= String text ]
, "unexpected" .= String text ]
RequestingShutdown reason ->
mconcat [ "kind" .= String "RequestingShutdown"
, "reason" .= String reason ]
ShutdownArmedAtSlot slot ->
mconcat [ "kind" .= String "ShutdownArmedAtSlot"
, "slot" .= toJSON slot ]
, "reason" .= String reason ]
ShutdownArmedAt lim ->
mconcat [ "kind" .= String "ShutdownArmedAt"
, "limit" .= lim ]

docShutdown :: Documented ShutdownTrace
docShutdown = addDocumentedNamespace [] docShutdown'
Expand All @@ -88,7 +88,7 @@ docShutdown' = Documented
[]
"Ringing the node shutdown doorbell"
, DocMsg
["ShutdownArmedAtSlot"]
["ShutdownArmedAt"]
[]
"Setting up node shutdown at given slot."
"Setting up node shutdown at given slot / block."
]
3 changes: 1 addition & 2 deletions cardano-node/src/Cardano/Tracing/OrphanInstances/Common.hs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ import Cardano.BM.Stats
import Cardano.BM.Tracing (HasPrivacyAnnotation (..), HasSeverityAnnotation (..),
Severity (..), ToObject (..), Tracer (..), TracingVerbosity (..),
Transformable (..))
import Cardano.Slotting.Block (BlockNo (..))
import Cardano.Node.Handlers.Shutdown ()
import Ouroboros.Consensus.Byron.Ledger.Block (ByronHash (..))
import Ouroboros.Consensus.HardFork.Combinator (OneEraHash (..))
import Ouroboros.Network.Block (HeaderHash, Tip (..))
Expand Down Expand Up @@ -102,7 +102,6 @@ instance ToJSON (OneEraHash xs) where
toJSON . Text.decodeLatin1 . B16.encode . SBS.fromShort $ bs

deriving newtype instance ToJSON ByronHash
deriving newtype instance ToJSON BlockNo

instance HasPrivacyAnnotation ResourceStats
instance HasSeverityAnnotation ResourceStats where
Expand Down
6 changes: 3 additions & 3 deletions cardano-node/test/Test/Cardano/Node/POM.hs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import Cardano.Tracing.Config (PartialTraceOptions (..), defaultPartia
partialTraceSelectionToEither)
import qualified Ouroboros.Consensus.Node as Consensus (NetworkP2PMode (..))
import Ouroboros.Consensus.Storage.LedgerDB.DiskPolicy (SnapshotInterval (..))
import Ouroboros.Network.Block (MaxSlotNo (..), SlotNo (..))
import Ouroboros.Network.Block (SlotNo (..))
import Ouroboros.Network.NodeToNode (AcceptedConnectionsLimit (..),
DiffusionMode (InitiatorAndResponderDiffusionMode))

Expand Down Expand Up @@ -84,7 +84,7 @@ testPartialCliConfig :: PartialNodeConfiguration
testPartialCliConfig =
PartialNodeConfiguration
{ pncSocketConfig = Last . Just $ SocketConfig mempty mempty mempty mempty
, pncShutdownConfig = Last . Just $ ShutdownConfig Nothing (Just . MaxSlotNo $ SlotNo 42)
, pncShutdownConfig = Last . Just $ ShutdownConfig Nothing (Just . ASlot . Identity $ SlotNo 42)
, pncConfigFile = mempty
, pncTopologyFile = mempty
, pncDatabaseFile = mempty
Expand Down Expand Up @@ -117,7 +117,7 @@ eExpectedConfig = do
(return $ PartialTracingOnLegacy defaultPartialTraceConfiguration)
return $ NodeConfiguration
{ ncSocketConfig = SocketConfig mempty mempty mempty mempty
, ncShutdownConfig = ShutdownConfig Nothing (Just . MaxSlotNo $ SlotNo 42)
, ncShutdownConfig = ShutdownConfig Nothing (Just . ASlot . Identity $ SlotNo 42)
, ncConfigFile = ConfigYamlFilePath "configuration/cardano/mainnet-config.json"
, ncTopologyFile = TopologyFile "configuration/cardano/mainnet-topology.json"
, ncDatabaseFile = DbFile "mainnet/db/"
Expand Down
8 changes: 4 additions & 4 deletions doc/new-tracing/tracers_doc_generated.md
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@
1. __Shutdown__
1. [AbnormalShutdown](#cardanonodeshutdownabnormalshutdown)
1. [RequestingShutdown](#cardanonodeshutdownrequestingshutdown)
1. [ShutdownArmedAtSlot](#cardanonodeshutdownshutdownarmedatslot)
1. [ShutdownArmedAtSlotBlock](#cardanonodeshutdownshutdownarmedatslotblock)
1. [ShutdownRequested](#cardanonodeshutdownshutdownrequested)
1. [ShutdownUnexpectedInput](#cardanonodeshutdownshutdownunexpectedinput)
1. __Startup__
Expand Down Expand Up @@ -8973,11 +8973,11 @@ Backends:
`EKGBackend`
Filtered by config value: `Notice`

### Cardano.Node.Shutdown.ShutdownArmedAtSlot
### Cardano.Node.Shutdown.ShutdownArmedAtSlotBlock


***
Setting up node shutdown at given slot.
Setting up node shutdown at given slot / block.
***


Expand Down Expand Up @@ -11610,4 +11610,4 @@ Basic information about this node collected at startup
Configuration: TraceConfig {tcOptions = fromList [(["Node"],[ConfSeverity {severity = Notice},ConfDetail {detail = DNormal},ConfBackend {backends = [Stdout MachineFormat,EKGBackend,Forwarder]}]),(["Node","AcceptPolicy"],[ConfSeverity {severity = Info}]),(["Node","BlockFetchClient","CompletedBlockFetch"],[ConfLimiter {maxFrequency = 2.0}]),(["Node","BlockFetchDecision"],[ConfSeverity {severity = Info}]),(["Node","BlockFetchServer","SendBlock"],[ConfSeverity {severity = Info}]),(["Node","ChainDB"],[ConfSeverity {severity = Info}]),(["Node","ChainDB","AddBlockEvent","AddBlockValidation","ValidCandidate"],[ConfLimiter {maxFrequency = 2.0}]),(["Node","ChainDB","AddBlockEvent","AddedBlockToQueue"],[ConfLimiter {maxFrequency = 2.0}]),(["Node","ChainDB","AddBlockEvent","AddedBlockToVolatileDB"],[ConfLimiter {maxFrequency = 2.0}]),(["Node","ChainDB","AddBlockEvent","AddedToCurrentChain"],[ConfSeverity {severity = Info}]),(["Node","ChainDB","AddBlockEvent","SwitchedToAFork"],[ConfSeverity {severity = Info}]),(["Node","ChainDB","CopyToImmutableDBEvent","CopiedBlockToImmutableDB"],[ConfLimiter {maxFrequency = 2.0}]),(["Node","ChainSyncServerHeader","ChainSyncServerEvent","ServerRead","RollBackward"],[ConfSeverity {severity = Info}]),(["Node","ChainSyncServerHeader","ChainSyncServerEvent","ServerRead","RollForward"],[ConfSeverity {severity = Info}]),(["Node","ChainSyncServerHeader","ChainSyncServerEvent","ServerRead","ServerRead"],[ConfSeverity {severity = Info}]),(["Node","ChainSyncServerHeader","ChainSyncServerEvent","ServerRead","ServerReadBlocked"],[ConfSeverity {severity = Info}]),(["Node","ConnectionManager","ConnectionManagerCounters"],[ConfSeverity {severity = Info}]),(["Node","DNSResolver"],[ConfSeverity {severity = Info}]),(["Node","DNSSubscription"],[ConfSeverity {severity = Info}]),(["Node","DiffusionInit"],[ConfSeverity {severity = Info}]),(["Node","ErrorPolicy"],[ConfSeverity {severity = Info}]),(["Node","Forge"],[ConfSeverity {severity = Info}]),(["Node","Forge","AdoptedBlock"],[ConfSeverity {severity = Info}]),(["Node","Forge","BlockContext"],[ConfSeverity {severity = Info}]),(["Node","Forge","BlockFromFuture"],[ConfSeverity {severity = Info}]),(["Node","Forge","DidntAdoptBlock"],[ConfSeverity {severity = Info}]),(["Node","Forge","ForgeStateUpdateError"],[ConfSeverity {severity = Info}]),(["Node","Forge","ForgedBlock"],[ConfSeverity {severity = Info}]),(["Node","Forge","ForgedInvalidBlock"],[ConfSeverity {severity = Info}]),(["Node","Forge","LedgerState"],[ConfSeverity {severity = Info}]),(["Node","Forge","LedgerView"],[ConfSeverity {severity = Info}]),(["Node","Forge","NoLedgerState"],[ConfSeverity {severity = Info}]),(["Node","Forge","NoLedgerView"],[ConfSeverity {severity = Info}]),(["Node","Forge","NodeCannotForge"],[ConfSeverity {severity = Info}]),(["Node","Forge","NodeIsLeader"],[ConfSeverity {severity = Info}]),(["Node","Forge","NodeNotLeader"],[ConfSeverity {severity = Info}]),(["Node","Forge","SlotIsImmutable"],[ConfSeverity {severity = Info}]),(["Node","Forge","StartLeadershipCheck"],[ConfSeverity {severity = Info}]),(["Node","Forge","StartLeadershipCheckPlus"],[ConfSeverity {severity = Info}]),(["Node","ForgeStats"],[ConfSeverity {severity = Info}]),(["Node","InboundGovernor","InboundGovernorCounters"],[ConfSeverity {severity = Info}]),(["Node","IpSubscription"],[ConfSeverity {severity = Info}]),(["Node","LocalConnectionManager","ConnectionManagerCounters"],[ConfSeverity {severity = Info}]),(["Node","LocalErrorPolicy"],[ConfSeverity {severity = Info}]),(["Node","LocalInboundGovernor","InboundGovernorCounters"],[ConfSeverity {severity = Info}]),(["Node","Mempool"],[ConfSeverity {severity = Info}]),(["Node","Mempool","AddedTx"],[ConfSeverity {severity = Info}]),(["Node","Mempool","ManuallyRemovedTxs"],[ConfSeverity {severity = Info}]),(["Node","Mempool","RejectedTx"],[ConfSeverity {severity = Info}]),(["Node","Mempool","RemoveTxs"],[ConfSeverity {severity = Info}]),(["Node","PeerSelectionCounters","PeerSelectionCounters"],[ConfSeverity {severity = Info}]),(["Node","Peers"],[ConfSeverity {severity = Info}]),(["Node","ReplayBlock","LedgerReplay"],[ConfSeverity {severity = Info}]),(["Node","Resources"],[ConfSeverity {severity = Info}]),(["Node","TxInbound","TxSubmissionCollected"],[ConfSeverity {severity = Info}]),(["Node","TxInbound","TxSubmissionProcessed"],[ConfSeverity {severity = Info}])], tcForwarder = TraceOptionForwarder {tofAddress = LocalSocket "/tmp/forwarder.sock", tofMode = Initiator, tofConnQueueSize = 2000, tofDisconnQueueSize = 200000, tofVerbosity = Minimum}, tcNodeName = Nothing, tcPeerFreqency = Just 2000, tcResourceFreqency = Just 5000}

682 log messages.
Generated at 2022-05-18 15:22:36.737189067 CEST.
Generated at 2022-05-18 15:22:36.737189067 CEST.
14 changes: 12 additions & 2 deletions nix/workbench/profiles/derived.jq
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@ def profile_name($p):
| era_defaults($p.era).generator as $generator_defaults
| era_defaults($p.era).composition as $composition_defaults
| era_defaults($p.era).node as $node_defaults
| $p.node.shutdown_on_slot_synced as $shutdown_slots
| $p.node.shutdown_on_block_synced as $shutdown_block
| ($p.node.shutdown_on_slot_synced // (($shutdown_block * 1.5 / $p.genesis.active_slots_coeff)
| ceil))
as $shutdown_slots
## Genesis
| [ "k\($p.composition.n_pools)" ]
+ if $p.composition.n_dense_hosts > 0
Expand Down Expand Up @@ -84,7 +87,10 @@ def add_derived_params:

## Absolute durations:
| ($gsis.epoch_length * $gsis.slot_duration) as $epoch_duration
| $node.shutdown_on_slot_synced as $shutdown_slots
| $node.shutdown_on_block_synced as $shutdown_block
| ($node.shutdown_on_slot_synced // (($shutdown_block * 1.5 / $gsis.active_slots_coeff)
| ceil))
as $shutdown_slots
| (if $shutdown_slots | type == "number"
then $shutdown_slots / $gsis.epoch_length | ceil
else $gtor.epochs
Expand Down Expand Up @@ -242,5 +248,9 @@ def profile_pretty_describe($p):
else [
" - terminate at slot: \($p.node.shutdown_on_slot_synced)"
] end
| . + if $p.node.shutdown_on_block_synced == null then []
else [
" - terminate at block: \($p.node.shutdown_on_block_synced)"
] end
| . + [""]
| join("\n");
Loading

0 comments on commit 5660b57

Please sign in to comment.