dkg/sync: support relay reconnect

ObolNetwork · Jul 4, 2022 · b00cb16 · b00cb16
1 parent 616f4da
commit b00cb16
Show file tree

Hide file tree

Showing 10 changed files with 62 additions and 6 deletions.
diff --git a/dkg/sync/client.go b/dkg/sync/client.go
@@ -79,7 +79,7 @@ func (c *Client) Run(ctx context.Context) error {
 		if err != nil {
 			return err
 		} else if reconnect {
-			log.Debug(ctx, "Outgoing sync dropped, reconnecting")
+			log.Debug(ctx, "Relay connection dropped, reconnecting")
 			continue
 		}
 
@@ -218,7 +218,7 @@ func (c *Client) connect(ctx context.Context, retry bool) (network.Stream, error
 }
 
 // isRelayError returns true if the error is due to temporary relay circuit recycling.
-func isRelayError(_ error) bool {
-	// TODO(corver): Detect circuit relay connection errors
-	return false
+func isRelayError(err error) bool {
+	return errors.Is(err, network.ErrReset) ||
+		errors.Is(err, network.ErrResourceScopeClosed)
 }
diff --git a/dkg/sync/server.go b/dkg/sync/server.go
@@ -204,7 +204,9 @@ func (s *Server) Start(ctx context.Context) {
 	s.tcpNode.SetStreamHandler(protocolID, func(stream network.Stream) {
 		ctx = log.WithCtx(ctx, z.Str("peer", p2p.PeerName(stream.Conn().RemotePeer())))
 		err := s.handleStream(ctx, stream)
-		if err != nil {
+		if isRelayError(err) {
+			log.Debug(ctx, "Relay connection dropped, expect reconnect")
+		} else if err != nil {
 			log.Warn(ctx, "Error serving sync protocol", err)
 		}
 	})

diff --git a/testutil/compose/compose/main.go b/testutil/compose/compose/main.go
@@ -268,6 +268,7 @@ func newNewCmd() *cobra.Command {
 	keygen := cmd.Flags().String("keygen", string(conf.KeyGen), "Key generation process: create, split, dkg")
 	buildLocal := cmd.Flags().Bool("build-local", conf.BuildBinary, "Enables building a local charon binary from source. Note this requires the CHARON_REPO env var.")
 	beaconNode := cmd.Flags().String("beacon-node", conf.BeaconNode, "Beacon node URL endpoint or 'mock' for simnet.")
+	extBootnode := cmd.Flags().String("external-bootnode", conf.BeaconNode, "Optional external bootnode HTTP url.")
 	splitKeys := cmd.Flags().String("split-keys-dir", conf.SplitKeysDir, "Directory containing keys to split for keygen==create, or empty not to split.")
 	featureSet := cmd.Flags().String("feature-set", conf.FeatureSet, "Minimum feature set to enable: alpha, beta, stable")
 
@@ -277,6 +278,7 @@ func newNewCmd() *cobra.Command {
 		conf.BeaconNode = *beaconNode
 		conf.SplitKeysDir = *splitKeys
 		conf.FeatureSet = *featureSet
+		conf.ExternalBootnode = *extBootnode
 
 		ctx := log.WithTopic(cmd.Context(), "new")
 		if err := compose.New(ctx, *dir, conf); err != nil {

diff --git a/testutil/compose/config.go b/testutil/compose/config.go
@@ -96,6 +96,9 @@ type Config struct {
 	// BeaconNode url endpoint or "mock" for simnet.
 	BeaconNode string `json:"beacon_node"`
 
+	// ExternalBootnode HTTP url endpoint or empty to disable.
+	ExternalBootnode string `json:"external_bootnode"`
+
 	// VCs define the types of validator clients to use.
 	VCs []VCType `json:"validator_clients"`
 

diff --git a/testutil/compose/lock.go b/testutil/compose/lock.go
@@ -109,6 +109,13 @@ func newNodeEnvs(index int, validatorMock bool, conf Config) []kv {
 		lockFile = fmt.Sprintf("/compose/node%d/cluster-lock.json", index)
 	}
 
+	p2pBootnodes := "http://bootnode:3640/enr"
+	p2pRelay := "false"
+	if conf.ExternalBootnode != "" {
+		p2pBootnodes = conf.ExternalBootnode
+		p2pRelay = "true"
+	}
+
 	return []kv{
 		{"data-dir", fmt.Sprintf("/compose/node%d", index)},
 		{"jaeger-service", fmt.Sprintf("node%d", index)},
@@ -120,7 +127,8 @@ func newNodeEnvs(index int, validatorMock bool, conf Config) []kv {
 		{"p2p-external-hostname", fmt.Sprintf("node%d", index)},
 		{"p2p-tcp-address", "0.0.0.0:3610"},
 		{"p2p_udp_address", "0.0.0.0:3630"},
-		{"p2p-bootnodes", "http://bootnode:3640/enr"},
+		{"p2p-bootnodes", p2pBootnodes},
+		{"p2p-bootnode-relay", fmt.Sprintf(`"%v"`, p2pRelay)},
 		{"beacon-node-endpoint", beaconNode},
 		{"simnet-validator-mock", fmt.Sprintf(`"%v"`, validatorMock)},
 		{"simnet-beacon_mock", fmt.Sprintf(`"%v"`, beaconMock)},

diff --git a/testutil/compose/testdata/TestDockerCompose_lock_dkg_template.golden b/testutil/compose/testdata/TestDockerCompose_lock_dkg_template.golden
@@ -52,6 +52,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""
@@ -123,6 +127,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""
@@ -194,6 +202,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""
@@ -265,6 +277,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""

diff --git a/testutil/compose/testdata/TestDockerCompose_lock_dkg_yml.golden b/testutil/compose/testdata/TestDockerCompose_lock_dkg_yml.golden
@@ -24,6 +24,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "true"
       CHARON_SIMNET_BEACON_MOCK: "true"
@@ -45,6 +46,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "true"
       CHARON_SIMNET_BEACON_MOCK: "true"
@@ -66,6 +68,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "true"
       CHARON_SIMNET_BEACON_MOCK: "true"
@@ -87,6 +90,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "true"
       CHARON_SIMNET_BEACON_MOCK: "true"

diff --git a/testutil/compose/testdata/TestDockerCompose_run_template.golden b/testutil/compose/testdata/TestDockerCompose_run_template.golden
@@ -52,6 +52,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""
@@ -123,6 +127,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""
@@ -194,6 +202,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""
@@ -265,6 +277,10 @@
      "Key": "p2p-bootnodes",
      "Value": "http://bootnode:3640/enr"
     },
+    {
+     "Key": "p2p-bootnode-relay",
+     "Value": "\"false\""
+    },
     {
      "Key": "beacon-node-endpoint",
      "Value": ""

diff --git a/testutil/compose/testdata/TestDockerCompose_run_yml.golden b/testutil/compose/testdata/TestDockerCompose_run_yml.golden
@@ -24,6 +24,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "false"
       CHARON_SIMNET_BEACON_MOCK: "true"
@@ -45,6 +46,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "false"
       CHARON_SIMNET_BEACON_MOCK: "true"
@@ -66,6 +68,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "true"
       CHARON_SIMNET_BEACON_MOCK: "true"
@@ -87,6 +90,7 @@ services:
       CHARON_P2P_TCP_ADDRESS: 0.0.0.0:3610
       CHARON_P2P_UDP_ADDRESS: 0.0.0.0:3630
       CHARON_P2P_BOOTNODES: http://bootnode:3640/enr
+      CHARON_P2P_BOOTNODE_RELAY: "false"
       CHARON_BEACON_NODE_ENDPOINT: 
       CHARON_SIMNET_VALIDATOR_MOCK: "false"
       CHARON_SIMNET_BEACON_MOCK: "true"

diff --git a/testutil/compose/testdata/TestNewDefaultConfig.golden b/testutil/compose/testdata/TestNewDefaultConfig.golden
@@ -10,6 +10,7 @@
  "key_gen": "create",
  "split_keys_dir": "",
  "beacon_node": "mock",
+ "external_bootnode": "",
  "validator_clients": [
   "teku",
   "lighthouse",