diff --git a/.changelog/2362.breaking.md b/.changelog/2362.breaking.md index 0f6d295d27e..ba6456f8f97 100644 --- a/.changelog/2362.breaking.md +++ b/.changelog/2362.breaking.md @@ -3,4 +3,5 @@ Simplify tendermint sentry node setup. Breaking configuration changes: - `worker.sentry.address` renamed to: `worker.registration.sentry.address` - `worker.sentry.cert_file` renamed to: `worker.registration.sentry.cert_file` -TODO: more inc +- `tendermint.private_peer_id` & `tendermint.persistent_peer` replaced by: +`tendermint.sentry.upstream_address` diff --git a/go/consensus/tendermint/tendermint.go b/go/consensus/tendermint/tendermint.go index 984a90d9584..675a7661072 100644 --- a/go/consensus/tendermint/tendermint.go +++ b/go/consensus/tendermint/tendermint.go @@ -76,10 +76,9 @@ const ( cfgABCIPruneStrategy = "tendermint.abci.prune.strategy" cfgABCIPruneNumKept = "tendermint.abci.prune.num_kept" - // CfgP2PPrivatePeerID configures tendermint's private peer ID(s). - CfgP2PPrivatePeerID = "tendermint.private_peer_id" - // CfgP2PPersistentPeer configures tendermint's persistent peer(s). - CfgP2PPersistentPeer = "tendermint.persistent_peer" + // CfgSentryUpstreamAddress defines nodes for which we act as a sentry for. + CfgSentryUpstreamAddress = "tendermint.sentry.upstream_address" + // CfgP2PDisablePeerExchange disables tendermint's peer-exchange (Pex) reactor. CfgP2PDisablePeerExchange = "tendermint.disable_peer_exchange" // CfgP2PSeeds configures tendermint's seed node(s). @@ -898,15 +897,6 @@ func (t *tendermintService) lazyInit() error { tenderConfig.TxIndex.Indexer = "null" tenderConfig.P2P.ListenAddress = viper.GetString(CfgCoreListenAddress) tenderConfig.P2P.ExternalAddress = viper.GetString(cfgCoreExternalAddress) - // Convert persistent peer IDs to lowercase (like other IDs) since - // Tendermint stores them in a map and uses a case sensitive string - // comparison to check ID equality. - tenderConfig.P2P.PrivatePeerIDs = strings.ToLower(strings.Join(viper.GetStringSlice(CfgP2PPrivatePeerID), ",")) - // Persistent peers need to be lowecase as p2p/transport.go:MultiplexTransport.upgrade() - // uses a case sensitive string comparision to validate public keys. - // Since persistent peers is expected to be in comma-delimited ID@host:port format, - // lowercasing the whole string is ok. - tenderConfig.P2P.PersistentPeers = strings.ToLower(strings.Join(viper.GetStringSlice(CfgP2PPersistentPeer), ",")) tenderConfig.P2P.PexReactor = !viper.GetBool(CfgP2PDisablePeerExchange) tenderConfig.P2P.SeedMode = viper.GetBool(CfgP2PSeedMode) // Seed Ids need to be Lowecase as p2p/transport.go:MultiplexTransport.upgrade() @@ -918,6 +908,33 @@ func (t *tendermintService) lazyInit() error { tenderConfig.P2P.AllowDuplicateIP = viper.GetBool(CfgDebugP2PAllowDuplicateIP) && cmflags.DebugDontBlameOasis() tenderConfig.RPC.ListenAddress = "" + sentryUpstreamAddrs := viper.GetStringSlice(CfgSentryUpstreamAddress) + if len(sentryUpstreamAddrs) > 0 { + t.Logger.Info("Acting as a tendermint sentry", "addrs", sentryUpstreamAddrs) + + // Persistent peers need to be lowecase as p2p/transport.go:MultiplexTransport.upgrade() + // uses a case sensitive string comparision to validate public keys. + // Since persistent peers is expected to be in comma-delimited ID@host:port format, + // lowercasing the whole string is ok. + tenderConfig.P2P.PersistentPeers = strings.ToLower(strings.Join(sentryUpstreamAddrs, ",")) + + var sentryUpstreamIDs []string + for _, addr := range sentryUpstreamAddrs { + parts := strings.Split(addr, "@") + if len(parts) != 2 { + return fmt.Errorf("malformed sentry upstream address: %s", addr) + } + sentryUpstreamIDs = append(sentryUpstreamIDs, parts[0]) + } + + // Convert persistent peer IDs to lowercase (like other IDs) since + // Tendermint stores them in a map and uses a case sensitive string + // comparison to check ID equality. + tenderConfig.P2P.PrivatePeerIDs = strings.ToLower(strings.Join(sentryUpstreamIDs, ",")) + + // XXX: tendermint v0.33 adds: `unconditional_peer_ids` which should also be set here. + } + if !tenderConfig.P2P.PexReactor { t.Logger.Info("pex reactor disabled", logging.LogEvent, api.LogEventPeerExchangeDisabled, @@ -1299,8 +1316,7 @@ func init() { Flags.String(cfgCoreExternalAddress, "", "tendermint address advertised to other nodes") Flags.String(cfgABCIPruneStrategy, abci.PruneDefault, "ABCI state pruning strategy") Flags.Int64(cfgABCIPruneNumKept, 3600, "ABCI state versions kept (when applicable)") - Flags.StringSlice(CfgP2PPrivatePeerID, []string{}, "Tendermint private peer(s) (i.e. they will not be gossiped to other peers) of the form ID") - Flags.StringSlice(CfgP2PPersistentPeer, []string{}, "Tendermint persistent peer(s) of the form ID@ip:port") + Flags.StringSlice(CfgSentryUpstreamAddress, []string{}, "Tendermint nodes for which we act as sentry of the form ID@ip:port") Flags.Bool(CfgP2PDisablePeerExchange, false, "Disable Tendermint's peer-exchange reactor") Flags.Bool(CfgP2PSeedMode, false, "run the tendermint node in seed mode") Flags.StringSlice(CfgP2PSeed, []string{}, "Tendermint seed node(s) of the form ID@host:port") diff --git a/go/oasis-test-runner/oasis/args.go b/go/oasis-test-runner/oasis/args.go index 2813a67ad4f..bdb5b4b6aea 100644 --- a/go/oasis-test-runner/oasis/args.go +++ b/go/oasis-test-runner/oasis/args.go @@ -82,19 +82,10 @@ func (args *argBuilder) tendermintCoreListenAddress(port uint16) *argBuilder { return args } -func (args *argBuilder) tendermintPersistentPeer(peers []string) *argBuilder { - for _, peer := range peers { - args.vec = append(args.vec, []string{ - "--" + tendermint.CfgP2PPersistentPeer, peer, - }...) - } - return args -} - -func (args *argBuilder) tendermintPrivatePeerID(peerIDs []string) *argBuilder { - for _, peerID := range peerIDs { +func (args *argBuilder) tendermintSentryUpstreamAddress(addrs []string) *argBuilder { + for _, addr := range addrs { args.vec = append(args.vec, []string{ - "--" + tendermint.CfgP2PPrivatePeerID, peerID, + "--" + tendermint.CfgSentryUpstreamAddress, addr, }...) } return args @@ -308,21 +299,12 @@ func (args *argBuilder) addSentries(sentries []*Sentry) *argBuilder { return args } -func (args *argBuilder) addSentriesAsPersistentPeers(sentries []*Sentry) *argBuilder { - var peers []string - for _, sentry := range sentries { - peers = append(peers, fmt.Sprintf("%s@127.0.0.1:%d", sentry.tmAddress, sentry.consensusPort)) - } - args = args.tendermintPersistentPeer(peers) - return args -} - -func (args *argBuilder) addValidatorsAsPrivatePeers(validators []*Validator) *argBuilder { - var peerIDs []string +func (args *argBuilder) addValidatorsAsSentryUpstreams(validators []*Validator) *argBuilder { + var addrs []string for _, val := range validators { - peerIDs = append(peerIDs, val.tmAddress) + addrs = append(addrs, fmt.Sprintf("%s@127.0.0.1:%d", val.tmAddress, val.consensusPort)) } - args = args.tendermintPrivatePeerID(peerIDs) + args = args.tendermintSentryUpstreamAddress(addrs) return args } diff --git a/go/oasis-test-runner/oasis/sentry.go b/go/oasis-test-runner/oasis/sentry.go index 9a0be4ea42f..79748717137 100644 --- a/go/oasis-test-runner/oasis/sentry.go +++ b/go/oasis-test-runner/oasis/sentry.go @@ -46,7 +46,7 @@ func (sentry *Sentry) startNode() error { workerSentryControlPort(sentry.controlPort). tendermintCoreListenAddress(sentry.consensusPort). appendNetwork(sentry.net). - addValidatorsAsPrivatePeers(validators) + addValidatorsAsSentryUpstreams(validators) if sentry.cmd, _, err = sentry.net.startOasisNode(sentry.dir, nil, args, sentry.Name, false, false); err != nil { return fmt.Errorf("oasis/sentry: failed to launch node %s: %w", sentry.Name, err) diff --git a/go/oasis-test-runner/oasis/validator.go b/go/oasis-test-runner/oasis/validator.go index 0a0cdc608f0..7337be1ed48 100644 --- a/go/oasis-test-runner/oasis/validator.go +++ b/go/oasis-test-runner/oasis/validator.go @@ -86,7 +86,6 @@ func (val *Validator) startNode() error { if len(val.sentries) > 0 { args = args.addSentries(val.sentries). - addSentriesAsPersistentPeers(val.sentries). tendermintDisablePeerExchange() }