diff --git a/ouroboros-network-api/CHANGELOG.md b/ouroboros-network-api/CHANGELOG.md index 4567f4ec5b8..d555c5f48ba 100644 --- a/ouroboros-network-api/CHANGELOG.md +++ b/ouroboros-network-api/CHANGELOG.md @@ -6,6 +6,8 @@ ### Non-Breaking changes +* Added `OutboundConnectionsState` data type + ## 0.7.1.0 -- 2024-03-14 ### Breaking changes diff --git a/ouroboros-network-api/ouroboros-network-api.cabal b/ouroboros-network-api/ouroboros-network-api.cabal index bcd543f666e..456749965bf 100644 --- a/ouroboros-network-api/ouroboros-network-api.cabal +++ b/ouroboros-network-api/ouroboros-network-api.cabal @@ -43,6 +43,7 @@ library Ouroboros.Network.PeerSelection.Bootstrap Ouroboros.Network.PeerSelection.LedgerPeers.Type + Ouroboros.Network.PeerSelection.LocalRootPeers Ouroboros.Network.PeerSelection.PeerMetric.Type Ouroboros.Network.PeerSelection.PeerAdvertise Ouroboros.Network.PeerSelection.PeerTrustable diff --git a/ouroboros-network-api/src/Ouroboros/Network/PeerSelection/LocalRootPeers.hs b/ouroboros-network-api/src/Ouroboros/Network/PeerSelection/LocalRootPeers.hs new file mode 100644 index 00000000000..e10653ca334 --- /dev/null +++ b/ouroboros-network-api/src/Ouroboros/Network/PeerSelection/LocalRootPeers.hs @@ -0,0 +1,21 @@ +{-# LANGUAGE DeriveGeneric #-} + +module Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState (..)) where + +import GHC.Generics +import NoThunks.Class + +data OutboundConnectionsState = + TrustedStateWithExternalPeers + -- ^ + -- * /in the Praos mode/: connected only to trusted local + -- peers and at least one bootstrap peer or public root; + -- * /in the Genesis mode/: meeting target of active big ledger peers; + -- * or it is in `Unrestricted` mode + -- (see `Ouroboros.Network.PeerSelection.Governor.AssociationMode`). + + | UntrustedState + -- ^ catch all other cases + deriving (Eq, Show, Generic) + +instance NoThunks OutboundConnectionsState diff --git a/ouroboros-network/CHANGELOG.md b/ouroboros-network/CHANGELOG.md index e80186f21ff..bc253a6bbbc 100644 --- a/ouroboros-network/CHANGELOG.md +++ b/ouroboros-network/CHANGELOG.md @@ -11,6 +11,11 @@ The counters cover more groups including: all peers, big ledger peers, bootstrap peers, local roots and shared peers. * `emptyPeerSelectionState` doesn't take targets of local roots. +* Added `daUpdateOutboundConnectionsState :: OutboundConnectionsState -> STM m ()` + to `Diffusion.Common.Applications`. This callback is to be provided by + consensus and is propagated all the way to the peer selection governor. +* Added `AssociationMode` and `LedgerStateJudgement` to `DebugPeerSelectionState`. + Both should be exposed through `EKG` counters by the node. ### Non-Breaking changes diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node.hs index 34f6e3ea9f3..2b79d3d0e9c 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node.hs @@ -94,6 +94,7 @@ import Simulation.Network.Snocket (AddressType (..), FD) import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers) import Ouroboros.Network.PeerSelection.LedgerPeers.Type (LedgerPeersConsensusInterface, UseLedgerPeers) +import Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState) import Ouroboros.Network.PeerSelection.PeerAdvertise (PeerAdvertise (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable) @@ -124,6 +125,8 @@ data Interfaces m = Interfaces , iDomainMap :: StrictTVar m (Map Domain [(IP, TTL)]) , iLedgerPeersConsensusInterface :: LedgerPeersConsensusInterface m + , iUpdateOutboundConnectionsState + :: OutboundConnectionsState -> STM m () } type NtNFD m = FD m NtNAddr @@ -410,6 +413,8 @@ run blockGeneratorArgs limits ni na tracersExtra tracerBlockFetch = , Node.aaShouldChainSyncExit = aShouldChainSyncExit na , Node.aaChainSyncEarlyExit = aChainSyncEarlyExit na , Node.aaOwnPeerSharing = aOwnPeerSharing na + , Node.aaUpdateOutboundConnectionsState = + iUpdateOutboundConnectionsState ni } --- Utils diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node/MiniProtocols.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node/MiniProtocols.hs index b3ec1c025bb..e79f6e158a4 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node/MiniProtocols.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Diffusion/Node/MiniProtocols.hs @@ -88,6 +88,7 @@ import Ouroboros.Network.NodeToNode (blockFetchMiniProtocolNum, chainSyncMiniProtocolNum, keepAliveMiniProtocolNum, peerSharingMiniProtocolNum) import Ouroboros.Network.PeerSelection.LedgerPeers +import Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState) import Ouroboros.Network.PeerSelection.PeerSharing qualified as PSTypes import Ouroboros.Network.PeerSharing (PeerSharingAPI, bracketPeerSharingClient, peerSharingClient, peerSharingServer) @@ -205,6 +206,8 @@ data AppArgs header block m = AppArgs , aaChainSyncEarlyExit :: Bool , aaOwnPeerSharing :: PSTypes.PeerSharing + , aaUpdateOutboundConnectionsState + :: OutboundConnectionsState -> STM m () } @@ -253,6 +256,7 @@ applications debugTracer nodeKernel , aaShouldChainSyncExit , aaChainSyncEarlyExit , aaOwnPeerSharing + , aaUpdateOutboundConnectionsState } toHeader = Diff.Applications @@ -270,6 +274,8 @@ applications debugTracer nodeKernel localResponderApp , Diff.daLedgerPeersCtx = aaLedgerPeersConsensusInterface + , Diff.daUpdateOutboundConnectionsState = + aaUpdateOutboundConnectionsState } where initiatorApp diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/LedgerPeers.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/LedgerPeers.hs index 2009a0e8540..3aae41ed196 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/LedgerPeers.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/LedgerPeers.hs @@ -86,6 +86,8 @@ instance Arbitrary ArbitraryLedgerStateJudgement where shrink (ArbitraryLedgerStateJudgement TooOld) = [] +-- TODO: import the `SlotNo` instance from +-- `Test.Ouroboros.Network.PeerSelection.Instances` newtype ArbitrarySlotNo = ArbitrarySlotNo { getArbitrarySlotNo :: SlotNo diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection.hs index 2b081ad4327..b53592f13ba 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection.hs @@ -31,6 +31,7 @@ module Test.Ouroboros.Network.PeerSelection import Control.Concurrent.Class.MonadSTM.Strict import Control.Exception (AssertionFailed (..), catch, evaluate) +import Control.Monad (when) import Control.Monad.Class.MonadTime.SI import Control.Monad.Class.MonadTimer.SI import Control.Tracer (Tracer (..)) @@ -62,6 +63,7 @@ import Ouroboros.Network.PeerSelection.Governor hiding (PeerSelectionState (..), peerSharing) import Ouroboros.Network.PeerSelection.Governor qualified as Governor import Ouroboros.Network.PeerSelection.LedgerPeers +import Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState) import Ouroboros.Network.PeerSelection.PeerAdvertise import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable (..)) @@ -188,6 +190,7 @@ tests = , testProperty "node uses ledger peers in non-sensitive mode" prop_governor_uses_ledger_peers ] + , testProperty "association mode" prop_governor_association_mode ] , testGroup "issues" [ testProperty "3233" prop_issue_3233 @@ -759,6 +762,7 @@ envEventCredits TraceEnvActivatePeer {} = 0 envEventCredits TraceEnvDeactivatePeer {} = 0 envEventCredits TraceEnvCloseConn {} = 0 +envEventCredits TraceEnvUseLedgerPeers {} = 30 envEventCredits TraceEnvSetLedgerStateJudgement {} = 30 envEventCredits TraceEnvSetUseBootstrapPeers {} = 30 @@ -3424,6 +3428,79 @@ prop_governor_uses_ledger_peers env = in counterexample (intercalate "\n" $ map show $ usesLedgerPeers) $ all snd usesLedgerPeers + +prop_governor_association_mode :: GovernorMockEnvironment -> Property +prop_governor_association_mode env = + let events = Signal.eventsFromListUpToTime (Time (10 * 60 * 60)) + . selectPeerSelectionTraceEvents + . runGovernorInMockEnvironment + $ env + + counters :: Signal (PeerSelectionSetsWithSizes PeerAddr) + counters = + selectGovState peerSelectionStateToView events + + -- accumulate local roots + localRoots :: Signal (Set PeerAddr) + localRoots = + Signal.keyedUntil + (\case + Just (GovernorEvent (TraceLocalRootPeersChanged a _)) -> LocalRootPeers.keysSet a + Just (MockEnvEvent (TraceEnvSetLocalRoots a)) -> LocalRootPeers.keysSet a + _ -> Set.empty + ) + (\_ -> Set.empty) + (\_ -> False) + . Signal.fromChangeEvents Nothing + . fmap Just + $ events + + publicRoots :: Signal (Set PeerAddr) + publicRoots = + Signal.keyedUntil + PublicRootPeers.toSet + (\_ -> Set.empty) + (\_ -> False) + . selectGovState Governor.publicRootPeers + $ events + + associationMode :: Signal AssociationMode + associationMode = + Signal.fromChangeEvents Unrestricted + $ selectGovAssociationMode events + + in counterexample (intercalate "\n" $ show <$> Signal.eventsToList events) + $ signalProperty 20 show + (\(cs, localRootSet, publicRootSet, am) -> + case am of + LocalRootsOnly -> + -- we need to remove local and public roots. They are changing + -- over time, and a node might keep using them, event though the + -- node is configured as an `Unrestricted` node. + -- + -- This makes this test only effective if a node starts in + -- `LocalRootsOnly` mode, until it is reconfigured. This can + -- discover some bugs in `readAssociationMode` but certainly not + -- all. + -- + -- TODO: write a more effective test. + Set.null (fst (viewKnownBootstrapPeers cs) + Set.\\ localRootSet + Set.\\ publicRootSet) + && Set.null (fst (viewKnownBigLedgerPeers cs) + Set.\\ localRootSet + Set.\\ publicRootSet) + && Set.null (fst (viewKnownNonRootPeers cs) + Set.\\ localRootSet + Set.\\ publicRootSet) + + Unrestricted -> True + ) + ((,,,) <$> counters + <*> localRoots + <*> publicRoots + <*> associationMode) + -- -- Utils for properties -- @@ -3442,6 +3519,18 @@ selectGovEvents = Signal.selectEvents (\case GovernorEvent e -> Just $! e _ -> Nothing) +selectGovCounters :: Events TestTraceEvent + -> Events PeerSelectionCounters +selectGovCounters = Signal.selectEvents + (\case GovernorCounters e -> Just $! e + _ -> Nothing) + +selectGovAssociationMode :: Events TestTraceEvent + -> Events AssociationMode +selectGovAssociationMode = Signal.selectEvents + (\case GovernorAssociationMode e -> Just $! e + _ -> Nothing) + selectGovState :: Eq a => (forall peerconn. Governor.PeerSelectionState PeerAddr peerconn -> a) -> Events TestTraceEvent @@ -3482,9 +3571,19 @@ _governorFindingPublicRoots :: Int -> STM IO UseBootstrapPeers -> STM IO LedgerStateJudgement -> PeerSharing + -> StrictTVar IO OutboundConnectionsState -> IO Void -_governorFindingPublicRoots targetNumberOfRootPeers readDomains readUseBootstrapPeers readLedgerStateJudgement peerSharing = do +_governorFindingPublicRoots targetNumberOfRootPeers readDomains readUseBootstrapPeers readLedgerStateJudgement peerSharing olocVar = do + countersVar <- newTVarIO emptyPeerSelectionCounters + publicStateVar <- makePublicPeerSelectionStateVar + debugStateVar <- newTVarIO $ emptyPeerSelectionState (mkStdGen 42) dnsSemaphore <- newLedgerAndPublicRootDNSSemaphore + let interfaces = PeerSelectionInterfaces { + countersVar, + publicStateVar, + debugStateVar, + readUseLedgerPeers = return DontUseLedgerPeers + } publicRootPeersProvider tracer (curry IP.toSockAddr) @@ -3492,20 +3591,15 @@ _governorFindingPublicRoots targetNumberOfRootPeers readDomains readUseBootstrap DNS.defaultResolvConf readDomains (ioDNSActions LookupReqAAndAAAA) $ \requestPublicRootPeers -> do - publicStateVar <- makePublicPeerSelectionStateVar - debugVar <- newTVarIO $ emptyPeerSelectionState (mkStdGen 42) - countersVar <- newTVarIO emptyPeerSelectionCounters peerSelectionGovernor tracer tracer tracer -- TODO: #3182 Rng seed should come from quickcheck. (mkStdGen 42) - countersVar - publicStateVar - debugVar actions { requestPublicRootPeers = \_ -> transformPeerSelectionAction requestPublicRootPeers } policy + interfaces where tracer :: Show a => Tracer IO a tracer = Tracer (BS.putStrLn . BS.pack . show) @@ -3527,8 +3621,12 @@ _governorFindingPublicRoots targetNumberOfRootPeers readDomains readUseBootstrap closePeerConnection = error "closePeerConnection" }, readUseBootstrapPeers, - readLedgerStateJudgement - } + readLedgerStateJudgement, + updateOutboundConnectionsState = \a -> do + a' <- readTVar olocVar + when (a /= a') $ + writeTVar olocVar a + } targets :: PeerSelectionTargets targets = nullPeerSelectionTargets { @@ -3590,6 +3688,7 @@ prop_issue_3550 = prop_governor_target_established_below defaultMaxTime $ pickColdPeersToForget = Script (PickFirst :| []), peerSharingFlag = PeerSharingEnabled, useBootstrapPeers = Script ((DontUseBootstrapPeers, NoDelay) :| []), + useLedgerPeers = Script ((UseLedgerPeers Always, NoDelay) :| []), ledgerStateJudgement = Script ((YoungEnough, NoDelay) :| []) } @@ -3626,6 +3725,7 @@ prop_issue_3515 = prop_governor_nolivelock $ pickColdPeersToForget = Script (PickFirst :| []), peerSharingFlag = PeerSharingEnabled, useBootstrapPeers = Script ((DontUseBootstrapPeers, NoDelay) :| []), + useLedgerPeers = Script ((UseLedgerPeers Always, NoDelay) :| []), ledgerStateJudgement = Script ((YoungEnough, NoDelay) :| []) } @@ -3662,6 +3762,7 @@ prop_issue_3494 = prop_governor_nofail $ pickColdPeersToForget = Script (PickFirst :| []), peerSharingFlag = PeerSharingEnabled, useBootstrapPeers = Script ((DontUseBootstrapPeers, NoDelay) :| []), + useLedgerPeers = Script ((UseLedgerPeers Always, NoDelay) :| []), ledgerStateJudgement = Script ((YoungEnough, NoDelay) :| []) } @@ -3714,6 +3815,7 @@ prop_issue_3233 = prop_governor_nolivelock $ pickColdPeersToForget = Script (PickFirst :| []), peerSharingFlag = PeerSharingEnabled, useBootstrapPeers = Script ((DontUseBootstrapPeers, NoDelay) :| []), + useLedgerPeers = Script ((UseLedgerPeers Always, NoDelay) :| []), ledgerStateJudgement = Script ((YoungEnough, NoDelay) :| []) } diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/Instances.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/Instances.hs index 80ce4b454d2..5023eeb3f78 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/Instances.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/Instances.hs @@ -1,6 +1,8 @@ +{-# LANGUAGE DerivingVia #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE StandaloneDeriving #-} {-# OPTIONS_GHC -Wno-orphans #-} @@ -16,19 +18,22 @@ module Test.Ouroboros.Network.PeerSelection.Instances ) where import Data.Text.Encoding (encodeUtf8) -import Data.Word (Word32) +import Data.Word (Word32, Word64) + +import Cardano.Slotting.Slot (SlotNo (..)) import Ouroboros.Network.PeerSelection.Governor import Data.Hashable import Data.IP qualified as IP +import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..)) +import Ouroboros.Network.PeerSelection.LedgerPeers.Type (AfterSlot (..), + UseLedgerPeers (..)) import Ouroboros.Network.PeerSelection.PeerAdvertise (PeerAdvertise (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) +import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable (..)) import Ouroboros.Network.PeerSelection.RelayAccessPoint (DomainAccessPoint (..), RelayAccessPoint (..)) - -import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..)) -import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable (..)) import Ouroboros.Network.Testing.Utils (ShrinkCarefully, prop_shrink_nonequal, prop_shrink_valid) import Test.QuickCheck @@ -51,7 +56,7 @@ instance Arbitrary PeerAddr where arbitrary = PeerAddr <$> arbitrarySizedNatural shrink _ = [] - +deriving via Word64 instance Arbitrary SlotNo instance Arbitrary PeerAdvertise where arbitrary = elements [ DoAdvertisePeer, DoNotAdvertisePeer ] @@ -64,6 +69,11 @@ instance Arbitrary PeerSharing where shrink PeerSharingDisabled = [] shrink PeerSharingEnabled = [PeerSharingDisabled] +instance Arbitrary AfterSlot where + arbitrary = oneof [ pure Always + , After <$> arbitrary + ] + instance Arbitrary UseBootstrapPeers where arbitrary = frequency [ (1, pure DontUseBootstrapPeers) , (1, UseBootstrapPeers <$> arbitrary) @@ -72,6 +82,12 @@ instance Arbitrary UseBootstrapPeers where shrink DontUseBootstrapPeers = [] shrink (UseBootstrapPeers _) = [DontUseBootstrapPeers] +instance Arbitrary UseLedgerPeers where + arbitrary = frequency + [ (2, pure DontUseLedgerPeers) + , (8, UseLedgerPeers <$> arbitrary) + ] + instance Arbitrary PeerTrustable where arbitrary = elements [ IsNotTrustable, IsTrustable ] diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/MockEnvironment.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/MockEnvironment.hs index 52befa2ca0a..1658cd74f22 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/MockEnvironment.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/PeerSelection/MockEnvironment.hs @@ -46,6 +46,7 @@ import System.Random (mkStdGen) import Control.Concurrent.Class.MonadSTM import Control.Concurrent.Class.MonadSTM.Strict qualified as StrictTVar import Control.Exception (throw) +import Control.Monad (when) import Control.Monad.Class.MonadAsync import Control.Monad.Class.MonadFork import Control.Monad.Class.MonadSay @@ -59,6 +60,7 @@ import Control.Tracer (Tracer (..), contramap, traceWith) import Ouroboros.Network.ExitPolicy import Ouroboros.Network.PeerSelection.Governor hiding (PeerSelectionState (..)) +import Ouroboros.Network.PeerSelection.Governor qualified as Governor import Ouroboros.Network.PeerSelection.State.LocalRootPeers qualified as LocalRootPeers import Ouroboros.Network.Testing.Data.Script (PickScript, Script (..), @@ -76,10 +78,9 @@ import Test.Ouroboros.Network.PeerSelection.PeerGraph import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..), requiresBootstrapPeers) -import Ouroboros.Network.PeerSelection.LedgerPeers (IsBigLedgerPeer, - LedgerPeersKind (..)) -import Ouroboros.Network.PeerSelection.LedgerPeers.Type - (LedgerStateJudgement (..)) +import Ouroboros.Network.PeerSelection.LedgerPeers +import Ouroboros.Network.PeerSelection.LocalRootPeers + (OutboundConnectionsState (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.PublicRootPeers (PublicRootPeers (..)) import Ouroboros.Network.PeerSelection.PublicRootPeers qualified as PublicRootPeers @@ -138,6 +139,7 @@ data GovernorMockEnvironment = GovernorMockEnvironment { pickColdPeersToForget :: !(PickScript PeerAddr), peerSharingFlag :: !PeerSharing, useBootstrapPeers :: !(TimedScript UseBootstrapPeers), + useLedgerPeers :: !(TimedScript UseLedgerPeers), ledgerStateJudgement :: !(TimedScript LedgerStateJudgement) } deriving (Show, Eq) @@ -209,25 +211,37 @@ governorAction mockEnv = do publicStateVar <- makePublicPeerSelectionStateVar lsjVar <- playTimedScript (contramap TraceEnvSetLedgerStateJudgement tracerMockEnv) (ledgerStateJudgement mockEnv) + lpVar <- playTimedScript (contramap TraceEnvUseLedgerPeers tracerMockEnv) + (useLedgerPeers mockEnv) usbVar <- playTimedScript (contramap TraceEnvSetUseBootstrapPeers tracerMockEnv) (useBootstrapPeers mockEnv) - debugVar <- StrictTVar.newTVarIO (emptyPeerSelectionState (mkStdGen 42)) + debugStateVar <- StrictTVar.newTVarIO (emptyPeerSelectionState (mkStdGen 42)) countersVar <- StrictTVar.newTVarIO emptyPeerSelectionCounters policy <- mockPeerSelectionPolicy mockEnv - actions <- mockPeerSelectionActions tracerMockEnv mockEnv (readTVar usbVar) (readTVar lsjVar) policy + actions <- mockPeerSelectionActions tracerMockEnv mockEnv + (readTVar usbVar) + (readTVar lpVar) + (readTVar lsjVar) + policy + let interfaces = PeerSelectionInterfaces { + countersVar, + publicStateVar, + debugStateVar, + -- TODO: peer selection tests are not relying on `UseLedgerPeers` + readUseLedgerPeers = return DontUseLedgerPeers + } + exploreRaces -- explore races within the governor _ <- forkIO $ do -- races with the governor should be explored labelThisThread "outbound-governor" _ <- peerSelectionGovernor tracerTracePeerSelection - tracerDebugPeerSelection + (tracerDebugPeerSelection <> traceAssociationMode interfaces actions) tracerTracePeerSelectionCounters (mkStdGen 42) - countersVar - publicStateVar - debugVar actions policy + interfaces atomically retry atomically retry -- block to allow the governor to run @@ -260,6 +274,7 @@ data TraceMockEnv = TraceEnvAddPeers !PeerGraph | TraceEnvPeersStatus !(Map PeerAddr PeerStatus) | TraceEnvSetUseBootstrapPeers !UseBootstrapPeers | TraceEnvSetLedgerStateJudgement !LedgerStateJudgement + | TraceEnvUseLedgerPeers !UseLedgerPeers deriving Show mockPeerSelectionActions :: forall m. @@ -268,6 +283,7 @@ mockPeerSelectionActions :: forall m. => Tracer m TraceMockEnv -> GovernorMockEnvironment -> STM m UseBootstrapPeers + -> STM m UseLedgerPeers -> STM m LedgerStateJudgement -> PeerSelectionPolicy PeerAddr m -> m (PeerSelectionActions PeerAddr (PeerConn m) m) @@ -279,6 +295,7 @@ mockPeerSelectionActions tracer targets } readUseBootstrapPeers + readUseLedgerPeers getLedgerStateJudgement policy = do scripts <- Map.fromList <$> @@ -301,12 +318,19 @@ mockPeerSelectionActions tracer v (\_ a -> TraceDynamic . TraceEnvPeersStatus <$> snapshotPeersStatus proxy a) return v + + onlyLocalOutboundConnsVar <- newTVarIO UntrustedState traceWith tracer (TraceEnvAddPeers peerGraph) traceWith tracer (TraceEnvSetLocalRoots localRootPeers) --TODO: make dynamic traceWith tracer (TraceEnvSetPublicRoots publicRootPeers) --TODO: make dynamic return $ mockPeerSelectionActions' tracer env policy - scripts targetsVar readUseBootstrapPeers getLedgerStateJudgement peerConns + scripts targetsVar + readUseBootstrapPeers + readUseLedgerPeers + getLedgerStateJudgement + peerConns + onlyLocalOutboundConnsVar where proxy :: Proxy m proxy = Proxy @@ -329,8 +353,10 @@ mockPeerSelectionActions' :: forall m. -> Map PeerAddr (TVar m PeerShareScript, TVar m PeerSharingScript, TVar m ConnectionScript) -> TVar m PeerSelectionTargets -> STM m UseBootstrapPeers + -> STM m UseLedgerPeers -> STM m LedgerStateJudgement -> TVar m (Map PeerAddr (TVar m PeerStatus)) + -> TVar m OutboundConnectionsState -> PeerSelectionActions PeerAddr (PeerConn m) m mockPeerSelectionActions' tracer GovernorMockEnvironment { @@ -342,8 +368,10 @@ mockPeerSelectionActions' tracer scripts targetsVar readUseBootstrapPeers + readUseLedgerPeers readLedgerStateJudgement - connsVar = + connsVar + outboundConnectionsStateVar = PeerSelectionActions { readLocalRootPeers = return (LocalRootPeers.toGroups localRootPeers), peerSharing = peerSharingFlag, @@ -360,7 +388,11 @@ mockPeerSelectionActions' tracer closePeerConnection }, readUseBootstrapPeers, - readLedgerStateJudgement + readLedgerStateJudgement, + updateOutboundConnectionsState = \a -> do + a' <- readTVar outboundConnectionsStateVar + when (a /= a') $ + writeTVar outboundConnectionsStateVar a } where -- TODO: make this dynamic @@ -376,6 +408,7 @@ mockPeerSelectionActions' tracer usingBootstrapPeers <- atomically $ requiresBootstrapPeers <$> readUseBootstrapPeers <*> readLedgerStateJudgement + useLedgerPeers <- atomically readUseLedgerPeers -- If the ledger state is YoungEnough we should get ledger peers. -- Otherwise we should get bootstrap peers let publicConfigPeers = PublicRootPeers.getPublicConfigPeers publicRootPeers @@ -385,17 +418,19 @@ mockPeerSelectionActions' tracer result = if usingBootstrapPeers then PublicRootPeers.fromBootstrapPeers bootstrapPeers - else case ledgerPeersKind of - AllLedgerPeers - | Set.null ledgerPeers -> - PublicRootPeers.fromPublicRootPeers publicConfigPeers - | otherwise -> - PublicRootPeers.fromLedgerPeers ledgerPeers - BigLedgerPeers - | Set.null ledgerPeers -> - PublicRootPeers.fromPublicRootPeers publicConfigPeers - | otherwise -> - PublicRootPeers.fromBigLedgerPeers bigLedgerPeers + else case useLedgerPeers of + DontUseLedgerPeers -> PublicRootPeers.empty + UseLedgerPeers _ -> case ledgerPeersKind of + AllLedgerPeers + | Set.null ledgerPeers -> + PublicRootPeers.fromPublicRootPeers publicConfigPeers + | otherwise -> + PublicRootPeers.fromLedgerPeers ledgerPeers + BigLedgerPeers + | Set.null ledgerPeers -> + PublicRootPeers.fromPublicRootPeers publicConfigPeers + | otherwise -> + PublicRootPeers.fromBigLedgerPeers bigLedgerPeers traceWith tracer (TraceEnvRootsResult (Set.toList (PublicRootPeers.toSet result))) return (result, ttl) @@ -585,10 +620,11 @@ mockPeerSelectionPolicy GovernorMockEnvironment { -- Utils for properties -- -data TestTraceEvent = GovernorDebug !(DebugPeerSelection PeerAddr) - | GovernorEvent !(TracePeerSelection PeerAddr) - | GovernorCounters !PeerSelectionCounters - | MockEnvEvent !TraceMockEnv +data TestTraceEvent = GovernorDebug !(DebugPeerSelection PeerAddr) + | GovernorEvent !(TracePeerSelection PeerAddr) + | GovernorCounters !PeerSelectionCounters + | GovernorAssociationMode !AssociationMode + | MockEnvEvent !TraceMockEnv -- Warning: be careful with writing properties that rely -- on trace events from both the governor and from the -- environment. These events typically occur in separate @@ -664,6 +700,16 @@ tracerTracePeerSelection = contramap f tracerTestTraceEvent tracerDebugPeerSelection :: Tracer (IOSim s) (DebugPeerSelection PeerAddr) tracerDebugPeerSelection = GovernorDebug `contramap` tracerTestTraceEvent +traceAssociationMode :: PeerSelectionInterfaces PeerAddr (PeerConn (IOSim s)) (IOSim s) + -> PeerSelectionActions PeerAddr (PeerConn (IOSim s)) (IOSim s) + -> Tracer (IOSim s) (DebugPeerSelection PeerAddr) +traceAssociationMode interfaces actions = Tracer $ \(TraceGovernorState _ _ st) -> do + associationMode <- atomically $ readAssociationMode + (readUseLedgerPeers interfaces) + (Governor.peerSharing actions) + (Governor.bootstrapPeersFlag st) + traceWith tracerTestTraceEvent (GovernorAssociationMode associationMode) + tracerTracePeerSelectionCounters :: Tracer (IOSim s) PeerSelectionCounters tracerTracePeerSelectionCounters = contramap GovernorCounters tracerTestTraceEvent @@ -746,6 +792,7 @@ instance Arbitrary GovernorMockEnvironment where pickColdPeersToForget <- arbitraryPickScript arbitrarySubsetOfPeers peerSharingFlag <- arbitrary useBootstrapPeers <- arbitrary + useLedgerPeers <- arbitrary ledgerStateJudgementList <- fmap getArbitraryLedgerStateJudgement <$> arbitrary ledgerStateJudgementDelays <- listOf1 (elements [NoDelay, ShortDelay]) let ledgerStateJudgementWithDelay = @@ -827,6 +874,7 @@ instance Arbitrary GovernorMockEnvironment where pickColdPeersToForget, peerSharingFlag, useBootstrapPeers, + useLedgerPeers, ledgerStateJudgement } = -- Special rule for shrinking the peerGraph because the localRootPeers @@ -869,6 +917,9 @@ instance Arbitrary GovernorMockEnvironment where ++ [ env { useBootstrapPeers = useBootstrapPeers' } | useBootstrapPeers' <- shrink useBootstrapPeers ] + ++ [ env { useLedgerPeers = useLedgerPeers' } + | useLedgerPeers' <- shrink useLedgerPeers + ] ++ [ env { ledgerStateJudgement = fmap (first getArbitraryLedgerStateJudgement) ledgerStateJudgement' } | ledgerStateJudgement' <- shrink (fmap (first ArbitraryLedgerStateJudgement) ledgerStateJudgement) ] diff --git a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Testnet/Simulation/Node.hs b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Testnet/Simulation/Node.hs index 21f1161a047..1233d3c7626 100644 --- a/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Testnet/Simulation/Node.hs +++ b/ouroboros-network/sim-tests-lib/Test/Ouroboros/Network/Testnet/Simulation/Node.hs @@ -34,7 +34,7 @@ module Test.Ouroboros.Network.Testnet.Simulation.Node import Control.Applicative (Alternative) import Control.Concurrent.Class.MonadMVar (MonadMVar) import Control.Concurrent.Class.MonadSTM.Strict -import Control.Monad (forM) +import Control.Monad (forM, when) import Control.Monad.Class.MonadAsync import Control.Monad.Class.MonadFork import Control.Monad.Class.MonadSay @@ -120,6 +120,8 @@ import Data.Typeable (Typeable) import Ouroboros.Network.BlockFetch (FetchMode (..), TraceFetchClientState, TraceLabelPeer (..)) import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..)) +import Ouroboros.Network.PeerSelection.LocalRootPeers + (OutboundConnectionsState (..)) import Ouroboros.Network.PeerSelection.PeerAdvertise (PeerAdvertise (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing) import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable) @@ -1066,6 +1068,7 @@ diffusionSimulation dMapVar = do chainSyncExitVar <- newTVarIO chainSyncExitOnBlockNo ledgerPeersVar <- initScript' ledgerPeers + onlyOutboundConnectionsStateVar <- newTVarIO UntrustedState let (bgaRng, rng) = Random.split $ mkStdGen seed acceptedConnectionsLimit = AcceptedConnectionsLimit maxBound maxBound 0 @@ -1147,6 +1150,11 @@ diffusionSimulation $ accPoolStake $ getLedgerPools $ ledgerPools) + , NodeKernel.iUpdateOutboundConnectionsState = + \a -> do + a' <- readTVar onlyOutboundConnectionsStateVar + when (a /= a') $ + writeTVar onlyOutboundConnectionsStateVar a } shouldChainSyncExit :: StrictTVar m (Maybe BlockNo) -> BlockHeader -> m Bool diff --git a/ouroboros-network/src/Ouroboros/Network/Diffusion/Common.hs b/ouroboros-network/src/Ouroboros/Network/Diffusion/Common.hs index b69bd29ed85..4a692fd821f 100644 --- a/ouroboros-network/src/Ouroboros/Network/Diffusion/Common.hs +++ b/ouroboros-network/src/Ouroboros/Network/Diffusion/Common.hs @@ -34,6 +34,7 @@ import Ouroboros.Network.NodeToNode qualified as NodeToNode import Ouroboros.Network.PeerSelection.Governor.Types (PublicPeerSelectionState) import Ouroboros.Network.PeerSelection.LedgerPeers.Type (LedgerPeersConsensusInterface) +import Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState) import Ouroboros.Network.Snocket (FileDescriptor) import Ouroboros.Network.Socket (SystemdSocketTracer) @@ -193,4 +194,13 @@ data Applications ntnAddr ntnVersion ntnVersionData -- -- TODO: it should be in 'InterfaceExtra' , daLedgerPeersCtx :: LedgerPeersConsensusInterface m + + -- | Callback provided by consensus to inform it if the node is + -- connected to only local roots or also some external peers. + -- + -- This is useful in order for the Bootstrap State Machine to + -- simply refuse to transition from TooOld to YoungEnough while + -- it only has local peers. + -- + , daUpdateOutboundConnectionsState :: OutboundConnectionsState -> STM m () } diff --git a/ouroboros-network/src/Ouroboros/Network/Diffusion/P2P.hs b/ouroboros-network/src/Ouroboros/Network/Diffusion/P2P.hs index e883a7c7544..c43dd854464 100644 --- a/ouroboros-network/src/Ouroboros/Network/Diffusion/P2P.hs +++ b/ouroboros-network/src/Ouroboros/Network/Diffusion/P2P.hs @@ -103,9 +103,9 @@ import Ouroboros.Network.PeerSelection.Governor qualified as Governor import Ouroboros.Network.PeerSelection.Governor.Types (ChurnMode (ChurnModeNormal), DebugPeerSelection (..), PeerSelectionActions, PeerSelectionCounters, - PeerSelectionPolicy (..), PeerSelectionState, - TracePeerSelection (..), emptyPeerSelectionCounters, - emptyPeerSelectionState) + PeerSelectionInterfaces (..), PeerSelectionPolicy (..), + PeerSelectionState, TracePeerSelection (..), + emptyPeerSelectionCounters, emptyPeerSelectionState) #ifdef POSIX import Ouroboros.Network.PeerSelection.Governor.Types (makeDebugPeerSelectionState) @@ -125,7 +125,6 @@ import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.PeerStateActions (PeerConnectionHandle, PeerSelectionActionsTrace (..), PeerStateActionsArguments (..), pchPeerSharing, withPeerStateActions) -import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable) import Ouroboros.Network.PeerSelection.RelayAccessPoint (RelayAccessPoint) import Ouroboros.Network.PeerSelection.RootPeersDNS import Ouroboros.Network.PeerSelection.RootPeersDNS.DNSActions (DNSActions, @@ -134,8 +133,7 @@ import Ouroboros.Network.PeerSelection.RootPeersDNS.LocalRootPeers (TraceLocalRootPeers) import Ouroboros.Network.PeerSelection.RootPeersDNS.PublicRootPeers (TracePublicRootPeers) -import Ouroboros.Network.PeerSelection.State.LocalRootPeers (HotValency, - WarmValency) +import Ouroboros.Network.PeerSelection.State.LocalRootPeers qualified as LocalRootPeers import Ouroboros.Network.PeerSharing (PeerSharingRegistry (..)) import Ouroboros.Network.RethrowPolicy import Ouroboros.Network.Server2 (ServerArguments (..), ServerTrace (..)) @@ -246,17 +244,17 @@ nullTracers = data ArgumentsExtra m = ArgumentsExtra { -- | selection targets for the peer governor -- - daPeerSelectionTargets :: PeerSelectionTargets + daPeerSelectionTargets :: PeerSelectionTargets - , daReadLocalRootPeers :: STM m [(HotValency, WarmValency, Map RelayAccessPoint (PeerAdvertise, PeerTrustable))] - , daReadPublicRootPeers :: STM m (Map RelayAccessPoint PeerAdvertise) + , daReadLocalRootPeers :: STM m (LocalRootPeers.Config RelayAccessPoint) + , daReadPublicRootPeers :: STM m (Map RelayAccessPoint PeerAdvertise) , daReadUseBootstrapPeers :: STM m UseBootstrapPeers -- | Peer's own PeerSharing value. -- -- This value comes from the node's configuration file and is static. - , daOwnPeerSharing :: PeerSharing - , daReadUseLedgerPeers :: STM m UseLedgerPeers + , daOwnPeerSharing :: PeerSharing + , daReadUseLedgerPeers :: STM m UseLedgerPeers -- | Timeout which starts once all responder protocols are idle. If the -- responders stay idle for duration of the timeout, the connection will @@ -267,7 +265,7 @@ data ArgumentsExtra m = ArgumentsExtra { -- -- See 'serverProtocolIdleTimeout'. -- - , daProtocolIdleTimeout :: DiffTime + , daProtocolIdleTimeout :: DiffTime -- | Time for which /node-to-node/ connections are kept in -- 'TerminatingState', it should correspond to the OS configured @TCP@ @@ -277,7 +275,7 @@ data ArgumentsExtra m = ArgumentsExtra { -- purpose is to be resilient for delayed packets in the same way @TCP@ -- is using @TIME_WAIT@. -- - , daTimeWaitTimeout :: DiffTime + , daTimeWaitTimeout :: DiffTime -- | Churn interval between churn events in deadline mode. A small fuzz -- is added (max 10 minutes) so that not all nodes churn at the same time. @@ -291,7 +289,7 @@ data ArgumentsExtra m = ArgumentsExtra { -- -- By default it is set to 300 seconds. -- - , daBulkChurnInterval :: DiffTime + , daBulkChurnInterval :: DiffTime } -- @@ -646,6 +644,7 @@ runM Interfaces , daLedgerPeersCtx = daLedgerPeersCtx@LedgerPeersConsensusInterface { lpGetLedgerStateJudgement } + , daUpdateOutboundConnectionsState } ApplicationsExtra { daRethrowPolicy @@ -983,7 +982,8 @@ runM Interfaces psReadUseBootstrapPeers = daReadUseBootstrapPeers, psPeerSharing = daOwnPeerSharing, psPeerConnToPeerSharing = pchPeerSharing diNtnPeerSharing, - psReadPeerSharingController = readTVar (getPeerSharingRegistry daPeerSharingRegistry) } + psReadPeerSharingController = readTVar (getPeerSharingRegistry daPeerSharingRegistry), + psUpdateOutboundConnectionsState = daUpdateOutboundConnectionsState } WithLedgerPeersArgs { wlpRng = ledgerPeersRng, wlpConsensusInterface = daLedgerPeersCtx, @@ -1004,11 +1004,15 @@ runM Interfaces peerSelectionTracer dtTracePeerSelectionCounters fuzzRng - countersVar - daPublicPeerSelectionVar - dbgVar peerSelectionActions peerSelectionPolicy + PeerSelectionInterfaces { + countersVar, + publicStateVar = daPublicPeerSelectionVar, + debugStateVar = dbgVar, + readUseLedgerPeers = daReadUseLedgerPeers + } + -- -- The peer churn governor: @@ -1194,9 +1198,15 @@ run tracers tracersExtra args argsExtra apps appsExtra = do (TrState state) ps <- readTVarIO dbgStateVar now <- getMonotonicTime - (up, bp) <- atomically $ (,) <$> upstreamyness metrics + (up, bp, lsj, am) <- atomically $ + (,,,) <$> upstreamyness metrics <*> fetchynessBlocks metrics - let dbgState = makeDebugPeerSelectionState ps up bp + <*> lpGetLedgerStateJudgement (daLedgerPeersCtx apps) + <*> Governor.readAssociationMode + (daReadUseLedgerPeers argsExtra) + (daOwnPeerSharing argsExtra) + (Governor.bootstrapPeersFlag ps) + let dbgState = makeDebugPeerSelectionState ps up bp lsj am traceWith (dtTracePeerSelectionTracer tracersExtra) (TraceDebugState now dbgState) ) diff --git a/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor.hs b/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor.hs index 62f222cd7e4..56d6e154a94 100644 --- a/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor.hs +++ b/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor.hs @@ -1,9 +1,15 @@ {-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE ScopedTypeVariables #-} +#if __GLASGOW_HASKELL__ < 904 +{-# OPTIONS_GHC -Wno-name-shadowing #-} +#endif + -- | This subsystem manages the discovery and selection of /upstream/ peers. -- module Ouroboros.Network.PeerSelection.Governor @@ -14,10 +20,13 @@ module Ouroboros.Network.PeerSelection.Governor PeerSelectionPolicy (..) , PeerSelectionTargets (..) , PeerSelectionActions (..) + , PeerSelectionInterfaces (..) , PeerStateActions (..) , TracePeerSelection (..) , ChurnAction (..) , DebugPeerSelection (..) + , AssociationMode (..) + , readAssociationMode , DebugPeerSelectionState (..) , peerSelectionGovernor -- * Peer churn governor @@ -43,6 +52,7 @@ module Ouroboros.Network.PeerSelection.Governor import Data.Foldable (traverse_) import Data.Hashable +import Data.Set qualified as Set import Data.Void (Void) import Control.Applicative (Alternative ((<|>))) @@ -56,6 +66,7 @@ import Control.Monad.Class.MonadTimer.SI import Control.Tracer (Tracer (..), traceWith) import System.Random +import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..)) import Ouroboros.Network.PeerSelection.Churn (ChurnCounters (..), peerChurnGovernor) import Ouroboros.Network.PeerSelection.Governor.ActivePeers qualified as ActivePeers @@ -65,8 +76,13 @@ import Ouroboros.Network.PeerSelection.Governor.KnownPeers qualified as KnownPee import Ouroboros.Network.PeerSelection.Governor.Monitor qualified as Monitor import Ouroboros.Network.PeerSelection.Governor.RootPeers qualified as RootPeers import Ouroboros.Network.PeerSelection.Governor.Types +import Ouroboros.Network.PeerSelection.LedgerPeers.Type (UseLedgerPeers (..)) +import Ouroboros.Network.PeerSelection.LocalRootPeers + (OutboundConnectionsState (..)) +import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing (..)) import Ouroboros.Network.PeerSelection.State.EstablishedPeers qualified as EstablishedPeers import Ouroboros.Network.PeerSelection.State.KnownPeers qualified as KnownPeers +import Ouroboros.Network.PeerSelection.State.LocalRootPeers qualified as LocalRootPeers {- $overview @@ -454,23 +470,20 @@ peerSelectionGovernor :: ( Alternative (STM m) -> Tracer m (DebugPeerSelection peeraddr) -> Tracer m PeerSelectionCounters -> StdGen - -> StrictTVar m PeerSelectionCounters - -> StrictTVar m (PublicPeerSelectionState peeraddr) - -> StrictTVar m (PeerSelectionState peeraddr peerconn) -> PeerSelectionActions peeraddr peerconn m -> PeerSelectionPolicy peeraddr m + -> PeerSelectionInterfaces peeraddr peerconn m -> m Void -peerSelectionGovernor tracer debugTracer countersTracer fuzzRng countersVar publicStateVar debugStateVar actions policy = +peerSelectionGovernor tracer debugTracer countersTracer fuzzRng + actions policy interfaces = JobPool.withJobPool $ \jobPool -> peerSelectionGovernorLoop tracer debugTracer countersTracer - countersVar - publicStateVar - debugStateVar actions policy + interfaces jobPool (emptyPeerSelectionState fuzzRng) @@ -502,22 +515,22 @@ peerSelectionGovernorLoop :: forall m peeraddr peerconn. => Tracer m (TracePeerSelection peeraddr) -> Tracer m (DebugPeerSelection peeraddr) -> Tracer m PeerSelectionCounters - -> StrictTVar m PeerSelectionCounters - -> StrictTVar m (PublicPeerSelectionState peeraddr) - -> StrictTVar m (PeerSelectionState peeraddr peerconn) -> PeerSelectionActions peeraddr peerconn m -> PeerSelectionPolicy peeraddr m + -> PeerSelectionInterfaces peeraddr peerconn m -> JobPool () m (Completion m peeraddr peerconn) -> PeerSelectionState peeraddr peerconn -> m Void peerSelectionGovernorLoop tracer debugTracer countersTracer - countersVar - publicStateVar - debugStateVar actions policy + interfaces@PeerSelectionInterfaces { + countersVar, + publicStateVar, + debugStateVar + } jobPool pst = do loop pst (Time 0) `catch` (\e -> traceWith tracer (TraceOutboundGovernorCriticalFailure e) >> throwIO e) @@ -558,13 +571,23 @@ peerSelectionGovernorLoop tracer -- get the current time after the governor returned from the blocking -- 'evalGuardedDecisions' call. now <- getMonotonicTime - let Decision { decisionTrace, decisionJobs, decisionState } = + + let Decision { decisionTrace, decisionJobs, decisionState = st'' } = timedDecision now mbCounters <- atomically $ do + -- Update outbound connections state + let peerSelectionView = peerSelectionStateToView st'' + associationMode <- readAssociationMode (readUseLedgerPeers interfaces) + (peerSharing actions) + (bootstrapPeersFlag st'') + updateOutboundConnectionsState + actions + (outboundConnectionsState associationMode peerSelectionView st'') + -- Update counters counters <- readTVar countersVar - let !counters' = peerSelectionStateToCounters decisionState + let !counters' = snd <$> peerSelectionView if counters' /= counters then writeTVar countersVar counters' >> return (Just counters') @@ -572,11 +595,11 @@ peerSelectionGovernorLoop tracer -- Trace counters traverse_ (traceWith countersTracer) mbCounters - + -- Trace peer selection traverse_ (traceWith tracer) decisionTrace mapM_ (JobPool.forkJob jobPool) decisionJobs - loop decisionState dbgUpdateAt' + loop st'' dbgUpdateAt' evalGuardedDecisions :: Time -> PeerSelectionState peeraddr peerconn @@ -667,3 +690,88 @@ wakeupDecision st _now = decisionState = st, decisionJobs = [] } + + +-- | Classify if a node is in promiscuous mode. +-- +-- A node is not in promiscuous mode only if: it doesn't use ledger peers, peer +-- sharing, the set of bootstrap peers is empty. +-- +readAssociationMode + :: MonadSTM m + => STM m UseLedgerPeers + -> PeerSharing + -> UseBootstrapPeers + -> STM m AssociationMode +readAssociationMode + readUseLedgerPeers + peerSharing + useBootstrapPeers + = + do useLedgerPeers <- readUseLedgerPeers + pure $ + case (useLedgerPeers, peerSharing, useBootstrapPeers) of + (DontUseLedgerPeers, PeerSharingDisabled, DontUseBootstrapPeers) + -> LocalRootsOnly + (DontUseLedgerPeers, PeerSharingDisabled, UseBootstrapPeers config) + | null config + -> LocalRootsOnly + _ -> Unrestricted + + +outboundConnectionsState + :: Ord peeraddr + => AssociationMode + -> PeerSelectionSetsWithSizes peeraddr + -> PeerSelectionState peeraddr peerconn + -> OutboundConnectionsState +outboundConnectionsState + associationMode + PeerSelectionView { + viewEstablishedPeers = (viewEstablishedPeers, _), + viewEstablishedBootstrapPeers = (viewEstablishedBootstrapPeers, _), + viewActiveBootstrapPeers = (viewActiveBootstrapPeers, _) + } + PeerSelectionState { + localRootPeers, + bootstrapPeersFlag + } + = + case (associationMode, bootstrapPeersFlag) of + {- + -- genesis mode + -- TODO: issue #4846 + (LocalRootsOnly, _) + | numberOfActiveBigLedgerPeers >= targetNumberOfActiveBigLedgerPeers + -> TrustedStateWithExternalPeers + + | otherwise + -> UntrustedState + -} + + (LocalRootsOnly, _) + | -- we are only connected to trusted local root + -- peers + viewEstablishedPeers `Set.isSubsetOf` trustableLocalRootSet + -> TrustedStateWithExternalPeers + + | otherwise + -> UntrustedState + + -- bootstrap mode + (Unrestricted, UseBootstrapPeers {}) + | -- we are only connected to trusted local root + -- peers or bootstrap peers + viewEstablishedPeers `Set.isSubsetOf` (viewEstablishedBootstrapPeers <> trustableLocalRootSet) + -- there's at least one active bootstrap peer + , not (Set.null viewActiveBootstrapPeers) + -> TrustedStateWithExternalPeers + + | otherwise + -> UntrustedState + + -- praos mode with public roots + (Unrestricted, DontUseBootstrapPeers) + -> UntrustedState + where + trustableLocalRootSet = LocalRootPeers.trustableKeysSet localRootPeers diff --git a/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor/Types.hs b/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor/Types.hs index 38e7936394b..419015f26a0 100644 --- a/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor/Types.hs +++ b/ouroboros-network/src/Ouroboros/Network/PeerSelection/Governor/Types.hs @@ -29,10 +29,12 @@ module Ouroboros.Network.PeerSelection.Governor.Types -- These records are needed to run the peer selection. , PeerStateActions (..) , PeerSelectionActions (..) + , PeerSelectionInterfaces (..) , ChurnMode (..) -- * P2P governor internals , PeerSelectionState (..) , emptyPeerSelectionState + , AssociationMode (..) , DebugPeerSelectionState (..) , makeDebugPeerSelectionState , assertPeerSelectionState @@ -143,10 +145,9 @@ import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..)) import Ouroboros.Network.PeerSelection.LedgerPeers (IsBigLedgerPeer, LedgerPeersKind) import Ouroboros.Network.PeerSelection.LedgerPeers.Type - (LedgerStateJudgement (..)) -import Ouroboros.Network.PeerSelection.PeerAdvertise (PeerAdvertise) + (LedgerStateJudgement (..), UseLedgerPeers (..)) +import Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing) -import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable) import Ouroboros.Network.PeerSelection.PublicRootPeers (PublicRootPeers) import Ouroboros.Network.PeerSelection.PublicRootPeers qualified as PublicRootPeers import Ouroboros.Network.PeerSelection.State.EstablishedPeers (EstablishedPeers) @@ -315,8 +316,6 @@ sanePeerSelectionTargets PeerSelectionTargets{..} = && targetNumberOfKnownBigLedgerPeers <= 10000 --- | Actions performed by the peer selection governor. --- -- These being pluggable allows: -- -- * choice of known peer root sets @@ -336,10 +335,7 @@ data PeerSelectionActions peeraddr peerconn m = PeerSelectionActions { -- It is structured as a collection of (non-overlapping) groups of peers -- where we are supposed to select n from each group. -- - readLocalRootPeers :: STM m [( HotValency - , WarmValency - , Map peeraddr ( PeerAdvertise - , PeerTrustable))], + readLocalRootPeers :: STM m (LocalRootPeers.Config peeraddr), readNewInboundConnection :: STM m (peeraddr, PeerSharing), @@ -386,9 +382,43 @@ data PeerSelectionActions peeraddr peerconn m = PeerSelectionActions { -- | Read the current ledger state judgement -- - readLedgerStateJudgement :: STM m LedgerStateJudgement + readLedgerStateJudgement :: STM m LedgerStateJudgement, + + -- | Callback provided by consensus to inform it if the node is + -- connected to only local roots or also some external peers. + -- + -- This is useful in order for the Bootstrap State Machine to + -- simply refuse to transition from TooOld to YoungEnough while + -- it only has local peers. + -- + updateOutboundConnectionsState :: OutboundConnectionsState -> STM m () + } +-- | Interfaces required by the peer selection governor, which do not need to +-- be shared with actions and thus are not part of `PeerSelectionActions`. +-- +data PeerSelectionInterfaces peeraddr peerconn m = PeerSelectionInterfaces { + -- | PeerSelectionCounters are shared with churn through a `StrictTVar`. + -- + countersVar :: StrictTVar m PeerSelectionCounters, + + -- | PublicPeerSelectionState var. + -- + publicStateVar :: StrictTVar m (PublicPeerSelectionState peeraddr), + + -- | PeerSelectionState shared for debugging purposes (to support SIGUSR1 + -- debug event tracing) + -- + debugStateVar :: StrictTVar m (PeerSelectionState peeraddr peerconn), + + -- | `UseLedgerPeers` used by `peerSelectionGovernor` to support + -- `HiddenRelayOrBP` + -- + readUseLedgerPeers :: STM m UseLedgerPeers + } + + -- | Callbacks which are performed to change peer state. -- data PeerStateActions peeraddr peerconn m = PeerStateActions { @@ -536,6 +566,27 @@ data PeerSelectionState peeraddr peerconn = PeerSelectionState { } deriving Show +-- | A node is classified as `LocalRootsOnly` if it is a hidden relay or +-- a BP, e.g. if it is configured such that it can only have a chance to be +-- connected to local roots. This is true if the node is configured in one of +-- two ways: +-- +-- * `DontUseBootstrapPeers`, `DontUseLedgerPeers` and +-- `PeerSharingDisabled`; or +-- * `UseBootstrapPeers`, `DontUseLedgerPeers` and +-- `PeerSharingDisabled`, but it's not using any bootstrap peers (i.e. it is +-- synced). +-- +-- Note that in the second case a node might transition between `LocalRootsOnly` +-- and `Unrestricted` modes, depending on `LedgerStateJudgement`. +-- +-- See `Ouroboros.Network.PeerSelection.Governor.readAssociationMode`. +-- +data AssociationMode = + LocalRootsOnly + | Unrestricted + deriving Show + ----------------------- -- Debug copy of Peer Selection State -- @@ -561,14 +612,18 @@ data DebugPeerSelectionState peeraddr = DebugPeerSelectionState { dpssInProgressDemoteHot :: !(Set peeraddr), dpssInProgressDemoteToCold :: !(Set peeraddr), dpssUpstreamyness :: !(Map peeraddr Int), - dpssFetchynessBlocks :: !(Map peeraddr Int) + dpssFetchynessBlocks :: !(Map peeraddr Int), + dpssLedgerStateJudgement :: !LedgerStateJudgement, + dpssAssociationMode :: !AssociationMode } deriving Show makeDebugPeerSelectionState :: PeerSelectionState peeraddr peerconn -> Map peeraddr Int -> Map peeraddr Int + -> LedgerStateJudgement + -> AssociationMode -> DebugPeerSelectionState peeraddr -makeDebugPeerSelectionState PeerSelectionState {..} up bp = +makeDebugPeerSelectionState PeerSelectionState {..} up bp lsj am = DebugPeerSelectionState { dpssTargets = targets , dpssLocalRootPeers = localRootPeers @@ -590,6 +645,8 @@ makeDebugPeerSelectionState PeerSelectionState {..} up bp = , dpssInProgressDemoteToCold = inProgressDemoteToCold , dpssUpstreamyness = up , dpssFetchynessBlocks = bp + , dpssLedgerStateJudgement = lsj + , dpssAssociationMode = am } -- | Public 'PeerSelectionState' that can be accessed by Peer Sharing @@ -622,8 +679,7 @@ makePublicPeerSelectionStateVar = newTVarIO emptyPublicPeerSelectionState -- toPublicState :: PeerSelectionState peeraddr peerconn -> PublicPeerSelectionState peeraddr -toPublicState PeerSelectionState { knownPeers - } = +toPublicState PeerSelectionState { knownPeers } = PublicPeerSelectionState { availableToShare = KnownPeers.getPeerSharingResponsePeers knownPeers diff --git a/ouroboros-network/src/Ouroboros/Network/PeerSelection/PeerSelectionActions.hs b/ouroboros-network/src/Ouroboros/Network/PeerSelection/PeerSelectionActions.hs index fd5fa8eed16..d3ad6ecbb65 100644 --- a/ouroboros-network/src/Ouroboros/Network/PeerSelection/PeerSelectionActions.hs +++ b/ouroboros-network/src/Ouroboros/Network/PeerSelection/PeerSelectionActions.hs @@ -37,6 +37,7 @@ import Ouroboros.Network.PeerSelection.Bootstrap (UseBootstrapPeers (..), requiresBootstrapPeers) import Ouroboros.Network.PeerSelection.Governor.Types import Ouroboros.Network.PeerSelection.LedgerPeers hiding (getLedgerPeers) +import Ouroboros.Network.PeerSelection.LocalRootPeers (OutboundConnectionsState) import Ouroboros.Network.PeerSelection.PeerAdvertise (PeerAdvertise (..)) import Ouroboros.Network.PeerSelection.PeerSharing (PeerSharing) import Ouroboros.Network.PeerSelection.PeerTrustable (PeerTrustable) @@ -65,8 +66,11 @@ data PeerSelectionActionsArgs peeraddr peerconn exception m = PeerSelectionActio -- ^ peer sharing configured value psPeerConnToPeerSharing :: peerconn -> PeerSharing, -- ^ Extract peer sharing information from peerconn - psReadPeerSharingController :: STM m (Map peeraddr (PeerSharingController peeraddr m)) + psReadPeerSharingController :: STM m (Map peeraddr (PeerSharingController peeraddr m)), -- ^ peer sharing registry + psUpdateOutboundConnectionsState + :: OutboundConnectionsState -> STM m () + -- ^ Callback which updates information about outbound connections state. } -- | Record of remaining parameters for withPeerSelectionActions @@ -111,7 +115,8 @@ withPeerSelectionActions psReadUseBootstrapPeers = useBootstrapped, psPeerSharing = sharing, psPeerConnToPeerSharing = peerConnToPeerSharing, - psReadPeerSharingController = sharingController } + psReadPeerSharingController = sharingController, + psUpdateOutboundConnectionsState = updateOutboundConnectionsState } ledgerPeersArgs PeerSelectionActionsDiffusionMode { psNewInboundConnections = readNewInboundConnections, psPeerStateActions = peerStateActions } k = do @@ -131,7 +136,8 @@ withPeerSelectionActions requestPeerShare, peerStateActions, readUseBootstrapPeers = useBootstrapped, - readLedgerStateJudgement = judgement } + readLedgerStateJudgement = judgement, + updateOutboundConnectionsState } withAsync (localRootPeersProvider localTracer diff --git a/ouroboros-network/src/Ouroboros/Network/PeerSelection/RootPeersDNS/LocalRootPeers.hs b/ouroboros-network/src/Ouroboros/Network/PeerSelection/RootPeersDNS/LocalRootPeers.hs index 4b5ad240fba..cc6c155125c 100644 --- a/ouroboros-network/src/Ouroboros/Network/PeerSelection/RootPeersDNS/LocalRootPeers.hs +++ b/ouroboros-network/src/Ouroboros/Network/PeerSelection/RootPeersDNS/LocalRootPeers.hs @@ -38,26 +38,19 @@ import Ouroboros.Network.PeerSelection.RootPeersDNS.DNSSemaphore (DNSSemaphore, newDNSLocalRootSemaphore, withDNSSemaphore) import Ouroboros.Network.PeerSelection.State.LocalRootPeers (HotValency, WarmValency) +import Ouroboros.Network.PeerSelection.State.LocalRootPeers qualified as LocalRootPeers data TraceLocalRootPeers peerAddr exception = - TraceLocalRootDomains [( HotValency - , WarmValency - , Map RelayAccessPoint (PeerAdvertise, PeerTrustable))] + TraceLocalRootDomains (LocalRootPeers.Config RelayAccessPoint) -- ^ 'Int' is the configured valency for the local producer groups | TraceLocalRootWaiting DomainAccessPoint DiffTime | TraceLocalRootResult DomainAccessPoint [(IP, DNS.TTL)] - | TraceLocalRootGroups [( HotValency - , WarmValency - , Map peerAddr (PeerAdvertise, PeerTrustable))] + | TraceLocalRootGroups (LocalRootPeers.Config peerAddr) -- ^ This traces the results of the local root peer provider | TraceLocalRootDNSMap (Map DomainAccessPoint [peerAddr]) -- ^ This traces the results of the domain name resolution - | TraceLocalRootReconfigured [( HotValency - , WarmValency - , Map RelayAccessPoint (PeerAdvertise, PeerTrustable))] -- ^ Old value - [( HotValency - , WarmValency - , Map RelayAccessPoint (PeerAdvertise, PeerTrustable))] -- ^ New value + | TraceLocalRootReconfigured (LocalRootPeers.Config RelayAccessPoint) -- ^ Old value + (LocalRootPeers.Config RelayAccessPoint) -- ^ New value | TraceLocalRootFailure DomainAccessPoint (DNSorIOError exception) --TODO: classify DNS errors, config error vs transitory | TraceLocalRootError DomainAccessPoint SomeException diff --git a/ouroboros-network/src/Ouroboros/Network/PeerSelection/State/LocalRootPeers.hs b/ouroboros-network/src/Ouroboros/Network/PeerSelection/State/LocalRootPeers.hs index a01a761392e..d8ea701c984 100644 --- a/ouroboros-network/src/Ouroboros/Network/PeerSelection/State/LocalRootPeers.hs +++ b/ouroboros-network/src/Ouroboros/Network/PeerSelection/State/LocalRootPeers.hs @@ -9,6 +9,7 @@ module Ouroboros.Network.PeerSelection.State.LocalRootPeers LocalRootPeers (..) , HotValency (..) , WarmValency (..) + , Config -- Export constructors for defining tests. , invariant -- * Basic operations @@ -23,6 +24,7 @@ module Ouroboros.Network.PeerSelection.State.LocalRootPeers , toGroupSets , toMap , keysSet + , trustableKeysSet -- * Special operations , clampToLimit , clampToTrustable @@ -71,6 +73,12 @@ newtype WarmValency = WarmValency { getWarmValency :: Int } deriving (Show, Eq, Ord) deriving Num via Int +-- | Data available from topology file. +-- +type Config peeraddr = + [(HotValency, WarmValency, Map peeraddr ( PeerAdvertise, PeerTrustable))] + + -- It is an abstract type, so the derived Show is unhelpful, e.g. for replaying -- test cases. -- @@ -251,3 +259,12 @@ isPeerTrustable peeraddr lrp = case Map.lookup peeraddr (toMap lrp) of Just (_, IsTrustable) -> True _ -> False + +trustableKeysSet :: LocalRootPeers peeraddr + -> Set peeraddr +trustableKeysSet (LocalRootPeers m _) = + Map.keysSet + . Map.filter (\(_, trustable) -> case trustable of + IsTrustable -> True + IsNotTrustable -> False) + $ m