go/tendermint: 'tendermint.sentry.upstream_address' flag

.changelog/2362.breaking.md

@@ -3,4 +3,7 @@ Simplify tendermint sentry node setup.
 Breaking configuration changes:
 - `worker.sentry.address` renamed to: `worker.registration.sentry.address`
 - `worker.sentry.cert_file` renamed to: `worker.registration.sentry.cert_file`
-TODO: more inc
+- `tendermint.private_peer_id` removed
+- added `tendermint.sentry.upstream_address` which sets both the
+`tendermint.private_peer_id` and `tendermint.peristent_peer` for the configured
+sentry addresses

go/consensus/tendermint/tendermint.go

@@ -76,8 +76,9 @@ const (
 	cfgABCIPruneStrategy = "tendermint.abci.prune.strategy"
 	cfgABCIPruneNumKept  = "tendermint.abci.prune.num_kept"
 
-	// CfgP2PPrivatePeerID configures tendermint's private peer ID(s).
-	CfgP2PPrivatePeerID = "tendermint.private_peer_id"
+	// CfgSentryUpstreamAddress defines nodes for which we act as a sentry for.
+	CfgSentryUpstreamAddress = "tendermint.sentry.upstream_address"
+
 	// CfgP2PPersistentPeer configures tendermint's persistent peer(s).
 	CfgP2PPersistentPeer = "tendermint.persistent_peer"
 	// CfgP2PDisablePeerExchange disables tendermint's peer-exchange (Pex) reactor.
@@ -898,16 +899,12 @@ func (t *tendermintService) lazyInit() error {
 	tenderConfig.TxIndex.Indexer = "null"
 	tenderConfig.P2P.ListenAddress = viper.GetString(CfgCoreListenAddress)
 	tenderConfig.P2P.ExternalAddress = viper.GetString(cfgCoreExternalAddress)
-	// Convert persistent peer IDs to lowercase (like other IDs) since
-	// Tendermint stores them in a map and uses a case sensitive string
-	// comparison to check ID equality.
-	tenderConfig.P2P.PrivatePeerIDs = strings.ToLower(strings.Join(viper.GetStringSlice(CfgP2PPrivatePeerID), ","))
+	tenderConfig.P2P.PexReactor = !viper.GetBool(CfgP2PDisablePeerExchange)
 	// Persistent peers need to be lowecase as p2p/transport.go:MultiplexTransport.upgrade()
 	// uses a case sensitive string comparision to validate public keys.
 	// Since persistent peers is expected to be in comma-delimited ID@host:port format,
 	// lowercasing the whole string is ok.
 	tenderConfig.P2P.PersistentPeers = strings.ToLower(strings.Join(viper.GetStringSlice(CfgP2PPersistentPeer), ","))
-	tenderConfig.P2P.PexReactor = !viper.GetBool(CfgP2PDisablePeerExchange)
 	tenderConfig.P2P.SeedMode = viper.GetBool(CfgP2PSeedMode)
 	// Seed Ids need to be Lowecase as p2p/transport.go:MultiplexTransport.upgrade()
 	// uses a case sensitive string comparision to validate public keys.
@@ -918,6 +915,31 @@ func (t *tendermintService) lazyInit() error {
 	tenderConfig.P2P.AllowDuplicateIP = viper.GetBool(CfgDebugP2PAllowDuplicateIP) && cmflags.DebugDontBlameOasis()
 	tenderConfig.RPC.ListenAddress = ""
 
+	sentryUpstreamAddrs := viper.GetStringSlice(CfgSentryUpstreamAddress)
+	if len(sentryUpstreamAddrs) > 0 {
+		t.Logger.Info("Acting as a tendermint sentry", "addrs", sentryUpstreamAddrs)
+
+		// Append sentry addresses to persistent peers.
+		tenderConfig.P2P.PersistentPeers = tenderConfig.P2P.PersistentPeers + "," +
+			strings.ToLower(strings.Join(sentryUpstreamAddrs, ","))
+
+		var sentryUpstreamIDs []string
+		for _, addr := range sentryUpstreamAddrs {
+			parts := strings.Split(addr, "@")
+			if len(parts) != 2 {
+				return fmt.Errorf("malformed sentry upstream address: %s", addr)
+			}
+			sentryUpstreamIDs = append(sentryUpstreamIDs, parts[0])
+		}
+
+		// Convert persistent peer IDs to lowercase (like other IDs) since
+		// Tendermint stores them in a map and uses a case sensitive string
+		// comparison to check ID equality.
+		tenderConfig.P2P.PrivatePeerIDs = strings.ToLower(strings.Join(sentryUpstreamIDs, ","))
+
+		// XXX: tendermint v0.33 adds: `unconditional_peer_ids` which should also be set here.
+	}
+
 	if !tenderConfig.P2P.PexReactor {
 		t.Logger.Info("pex reactor disabled",
 			logging.LogEvent, api.LogEventPeerExchangeDisabled,
@@ -1299,7 +1321,7 @@ func init() {
 	Flags.String(cfgCoreExternalAddress, "", "tendermint address advertised to other nodes")
 	Flags.String(cfgABCIPruneStrategy, abci.PruneDefault, "ABCI state pruning strategy")
 	Flags.Int64(cfgABCIPruneNumKept, 3600, "ABCI state versions kept (when applicable)")
-	Flags.StringSlice(CfgP2PPrivatePeerID, []string{}, "Tendermint private peer(s) (i.e. they will not be gossiped to other peers) of the form ID")
+	Flags.StringSlice(CfgSentryUpstreamAddress, []string{}, "Tendermint nodes for which we act as sentry of the form ID@ip:port")
 	Flags.StringSlice(CfgP2PPersistentPeer, []string{}, "Tendermint persistent peer(s) of the form ID@ip:port")
 	Flags.Bool(CfgP2PDisablePeerExchange, false, "Disable Tendermint's peer-exchange reactor")
 	Flags.Bool(CfgP2PSeedMode, false, "run the tendermint node in seed mode")

go/oasis-test-runner/oasis/args.go

@@ -82,19 +82,10 @@ func (args *argBuilder) tendermintCoreListenAddress(port uint16) *argBuilder {
 	return args
 }
 
-func (args *argBuilder) tendermintPersistentPeer(peers []string) *argBuilder {
-	for _, peer := range peers {
-		args.vec = append(args.vec, []string{
-			"--" + tendermint.CfgP2PPersistentPeer, peer,
-		}...)
-	}
-	return args
-}
-
-func (args *argBuilder) tendermintPrivatePeerID(peerIDs []string) *argBuilder {
-	for _, peerID := range peerIDs {
+func (args *argBuilder) tendermintSentryUpstreamAddress(addrs []string) *argBuilder {
+	for _, addr := range addrs {
 		args.vec = append(args.vec, []string{
-			"--" + tendermint.CfgP2PPrivatePeerID, peerID,
+			"--" + tendermint.CfgSentryUpstreamAddress, addr,
 		}...)
 	}
 	return args
@@ -308,21 +299,12 @@ func (args *argBuilder) addSentries(sentries []*Sentry) *argBuilder {
 	return args
 }
 
-func (args *argBuilder) addSentriesAsPersistentPeers(sentries []*Sentry) *argBuilder {
-	var peers []string
-	for _, sentry := range sentries {
-		peers = append(peers, fmt.Sprintf("%[email protected]:%d", sentry.tmAddress, sentry.consensusPort))
-	}
-	args = args.tendermintPersistentPeer(peers)
-	return args
-}
-
-func (args *argBuilder) addValidatorsAsPrivatePeers(validators []*Validator) *argBuilder {
-	var peerIDs []string
+func (args *argBuilder) addValidatorsAsSentryUpstreams(validators []*Validator) *argBuilder {
+	var addrs []string
 	for _, val := range validators {
-		peerIDs = append(peerIDs, val.tmAddress)
+		addrs = append(addrs, fmt.Sprintf("%[email protected]:%d", val.tmAddress, val.consensusPort))
 	}
-	args = args.tendermintPrivatePeerID(peerIDs)
+	args = args.tendermintSentryUpstreamAddress(addrs)
 	return args
 }
 

go/oasis-test-runner/oasis/sentry.go

@@ -46,7 +46,7 @@ func (sentry *Sentry) startNode() error {
 		workerSentryControlPort(sentry.controlPort).
 		tendermintCoreListenAddress(sentry.consensusPort).
 		appendNetwork(sentry.net).
-		addValidatorsAsPrivatePeers(validators)
+		addValidatorsAsSentryUpstreams(validators)
 
 	if sentry.cmd, _, err = sentry.net.startOasisNode(sentry.dir, nil, args, sentry.Name, false, false); err != nil {
 		return fmt.Errorf("oasis/sentry: failed to launch node %s: %w", sentry.Name, err)

go/oasis-test-runner/oasis/validator.go

@@ -86,7 +86,6 @@ func (val *Validator) startNode() error {
 
 	if len(val.sentries) > 0 {
 		args = args.addSentries(val.sentries).
-			addSentriesAsPersistentPeers(val.sentries).
 			tendermintDisablePeerExchange()
 	}