core: Retry remote core client three times before failing out (#344)

core/client.go

@@ -3,6 +3,8 @@ package core
 import (
 	"fmt"
 
+	retryhttp "github.com/hashicorp/go-retryablehttp"
+
 	corenode "github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/rpc/client"
 	"github.com/tendermint/tendermint/rpc/client/http"
@@ -14,9 +16,13 @@ type Client = client.Client
 
 // NewRemote creates a new Client that communicates with a remote Core endpoint over HTTP.
 func NewRemote(protocol, remoteAddr string) (Client, error) {
-	return http.New(
+	httpClient := retryhttp.NewClient()
+	httpClient.RetryMax = 2
+
+	return http.NewWithClient(
 		fmt.Sprintf("%s://%s", protocol, remoteAddr),
 		"/websocket",
+		httpClient.StandardClient(),
 	)
 }
 

core/client_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -102,3 +103,21 @@ func TestRemoteClient_StartBlockSubscription_And_GetBlock(t *testing.T) {
 	require.NoError(t, client.Stop())
 	require.NoError(t, remote.Stop())
 }
+
+// TestRemoteClient_RetryDial ensures 3 additional attempts
+// are made to dial the remote node.
+func TestRemoteClient_RetryDial(t *testing.T) {
+	remote := StartMockNode()
+	protocol, ip := getRemoteEndpoint(remote)
+
+	// deliberately stop remote
+	err := remote.Stop()
+	require.NoError(t, err)
+
+	client, err := NewRemote(protocol, ip)
+	require.NoError(t, err)
+
+	_, err = client.Block(context.Background(), nil)
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "giving up after 3 attempt(s)")
+}

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/celestiaorg/nmt v0.8.0
 	github.com/celestiaorg/rsmt2d v0.3.0
 	github.com/gogo/protobuf v1.3.2
+	github.com/hashicorp/go-retryablehttp v0.7.1-0.20211018174820-ff6d014e72d9
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/ipfs/go-bitswap v0.4.0
 	github.com/ipfs/go-block-format v0.0.3

go.sum

@@ -454,7 +454,9 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
@@ -464,7 +466,10 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-retryablehttp v0.7.1-0.20211018174820-ff6d014e72d9 h1:RSp277I12pTdb5wiZWy9qtE0IN4oC4BH5a/LIcNxw0w=
+github.com/hashicorp/go-retryablehttp v0.7.1-0.20211018174820-ff6d014e72d9/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=