utils: Add grpc middleware for connection retry

Gopkg.toml

@@ -57,6 +57,10 @@
   name = "gopkg.in/yaml.v2"
   version = "v2.1.1"
 
+[[constraint]]
+  name = "github.com/grpc-ecosystem/go-grpc-middleware"
+  version = "1.0.0"
+
 [prune]
   go-tests = true
   unused-packages = true

cmd/csi-sanity/Makefile

@@ -1,5 +1,5 @@
 APP_NAME := csi-sanity
-VER :=$(shell git describe)
+VER :=$(shell git describe --tags --dirty)
 RELEASEVER := $(shell git describe --abbrev=0)
 BRANCH := $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
 SHA := $(shell git rev-parse --short HEAD)

utils/grpcutil.go

@@ -23,10 +23,13 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 )
 
+const maxretries int = 3
+
 // Connect address by grpc
 func Connect(address string) (*grpc.ClientConn, error) {
 	dialOptions := []grpc.DialOption{
@@ -41,16 +44,25 @@ func Connect(address string) (*grpc.ClientConn, error) {
 				}))
 	}
 
+	// Add retry support.
+	dialOptions = append(dialOptions, grpc.WithUnaryInterceptor(grpc_retry.UnaryClientInterceptor()))
+
 	conn, err := grpc.Dial(address, dialOptions...)
 	if err != nil {
 		return nil, err
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
+
+	// Retry a few times before returning connection error.
+	retries := 0
 	for {
 		if !conn.WaitForStateChange(ctx, conn.GetState()) {
-			return conn, fmt.Errorf("Connection timed out")
+			retries++
+			if retries >= maxretries {
+				return conn, fmt.Errorf("Connection timed out")
+			}
 		}
 		if conn.GetState() == connectivity.Ready {
 			return conn, nil