Merge pull request #3 from heetch/rog-003-retry

retry on failure to connect to Kafka

client.go

@@ -8,8 +8,10 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/Shopify/sarama"
+	"gopkg.in/retry.v1"
 )
 
 var ErrDisabled = fmt.Errorf("kafka tests are disabled")
@@ -35,6 +37,10 @@ var ErrDisabled = fmt.Errorf("kafka tests are disabled")
 //		A boolean as parsed by strconv.ParseBool. If this
 //		is true, a secure TLS connection will be used.
 //
+//	- $KAFKA_TIMEOUT
+//		The maximum duration to wait when trying to connect
+//		to Kakfa. Defaults to "30s".
+//
 // The returned Kafka instance must be closed after use.
 func New() (*Kafka, error) {
 	disabled, err := boolVar("KAFKA_DISABLE")
@@ -59,11 +65,29 @@ func New() (*Kafka, error) {
 		saslUser:     os.Getenv("KAFKA_USERNAME"),
 		saslPassword: os.Getenv("KAFKA_PASSWORD"),
 	}
-	admin, err := sarama.NewClusterAdmin(addrs, client.Config())
-	if err != nil {
-		return nil, fmt.Errorf("cannot connect to Kafka cluster at %q: %v", addrs, err)
+	// The cluster might not be available immediately, so try
+	// for a while before giving up.
+	retryLimit := 30 * time.Second
+	if limit := os.Getenv("KAFKA_TIMEOUT"); limit != "" {
+		retryLimit, err = time.ParseDuration(limit)
+		if err != nil {
+			return nil, fmt.Errorf("bad value for KAFKA_TIMEOUT: %v", err)
+		}
+	}
+	retryStrategy := retry.LimitTime(retryLimit, retry.Exponential{
+		Initial:  time.Millisecond,
+		MaxDelay: time.Second,
+	})
+	for a := retry.Start(retryStrategy, nil); a.Next(); {
+		admin, err := sarama.NewClusterAdmin(addrs, client.Config())
+		if err == nil {
+			client.admin = admin
+			break
+		}
+		if !a.Next() {
+			return nil, fmt.Errorf("cannot connect to Kafka cluster at %q after %v: %v", addrs, retryLimit, err)
+		}
 	}
-	client.admin = admin
 	return client, nil
 }
 

go.mod

@@ -6,4 +6,5 @@ require (
 	github.com/Shopify/sarama v1.24.1
 	github.com/frankban/quicktest v1.4.1
 	github.com/klauspost/cpuid v1.2.1 // indirect
+	gopkg.in/retry.v1 v1.0.3
 )

go.sum

@@ -13,10 +13,12 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20=
 github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg=
 github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
@@ -38,6 +40,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM=
+github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -61,3 +65,5 @@ gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc440
 gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs=
+gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g=