[6.1.x] configurable HTTP timeout. (#123)

Updates https://github.com/gravitational/gravity.e/issues/4222.

monitoring/checkers.go

@@ -85,13 +85,17 @@ func TimeDriftHealth(config TimeDriftCheckerConfig) (c health.Checker, err error
 func EtcdHealth(config *ETCDConfig) (health.Checker, error) {
 	const name = "etcd-healthz"
 
-	transport, err := config.NewHTTPTransport()
-	if err != nil {
-		return nil, trace.Wrap(err)
+	client := config.Client
+	if client == nil {
+		var err error
+		client, err = config.NewClient()
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
 	}
 	createChecker := func(addr string) (health.Checker, error) {
 		endpoint := fmt.Sprintf("%v/health", addr)
-		return NewHTTPHealthzCheckerWithTransport(name, endpoint, transport, etcdChecker), nil
+		return NewHTTPHealthzCheckerWithClient(name, endpoint, client, etcdChecker), nil
 	}
 	var checkers []health.Checker
 	for _, endpoint := range config.Endpoints {

monitoring/etcd.go

@@ -46,8 +46,14 @@ type ETCDConfig struct {
 	// InsecureSkipVerify controls whether a client verifies the
 	// server's certificate chain and host name.
 	InsecureSkipVerify bool
+	// Client specifies the optional HTTP client to use for checks.
+	// If unspecified, one will be created with default settings
+	Client *http.Client
 }
 
+// defaultHTTPTimeout defines the default client timeout for HTTP-based checks
+const defaultHTTPTimeout = 10 * time.Second
+
 // defaultTLSHandshakeTimeout specifies the default maximum amount of time
 // spent waiting to for a TLS handshake
 const defaultTLSHandshakeTimeout = 10 * time.Second
@@ -95,6 +101,18 @@ func etcdStatus(payload []byte) (healthy bool, err error) {
 	return (result.Health == "true" || nresult.Health == true), nil
 }
 
+// NewClient returns a new HTTP client with default configuration
+func (r *ETCDConfig) NewClient() (*http.Client, error) {
+	transport, err := r.NewHTTPTransport()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return &http.Client{
+		Transport: transport,
+		Timeout:   defaultHTTPTimeout,
+	}, nil
+}
+
 // NewHTTPTransport creates a new http.Transport from the specified
 // set of attributes.
 // The resulting transport can be used to create an http.Client

monitoring/healthz.go

@@ -22,15 +22,12 @@ import (
 	"net"
 	"net/http"
 	"strconv"
-	"time"
 
 	"github.com/gravitational/satellite/agent/health"
 	pb "github.com/gravitational/satellite/agent/proto/agentpb"
 	"github.com/gravitational/trace"
 )
 
-const healthzCheckTimeout = 1 * time.Second
-
 // HTTPResponseChecker is a function that can validate service health
 // from the provided response
 type HTTPResponseChecker func(response io.Reader) error
@@ -103,8 +100,14 @@ func NewUnixSocketHealthzChecker(name, URL, socketPath string, checker HTTPRespo
 func NewHTTPHealthzCheckerWithTransport(name, URL string, transport http.RoundTripper, checker HTTPResponseChecker) health.Checker {
 	client := &http.Client{
 		Transport: transport,
-		Timeout:   healthzCheckTimeout,
+		Timeout:   defaultHTTPTimeout,
 	}
+	return NewHTTPHealthzCheckerWithClient(name, URL, client, checker)
+}
+
+// NewHTTPHealthzCheckerWithClient creates a health.Checker for an HTTP health endpoint
+// using the specified HTTP client, URL and a custom response checker
+func NewHTTPHealthzCheckerWithClient(name, URL string, client *http.Client, checker HTTPResponseChecker) health.Checker {
 	return &HTTPHealthzChecker{
 		name:    name,
 		URL:     URL,

monitoring/healthz_test.go

@@ -77,10 +77,14 @@ func (_ *HealthzSuite) TestObtainsSuccessProbe(c *C) {
 func (_ *HealthzSuite) TestUsesClientTimeout(c *C) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Response time is too long
-		time.Sleep(healthzCheckTimeout * 2)
+		time.Sleep(2 * time.Second)
 	}))
 	defer srv.Close()
-	checker := NewHTTPHealthzChecker("foo", srv.URL, func(resp io.Reader) error {
+	client := &http.Client{
+		Transport: http.RoundTripper(nil),
+		Timeout:   1 * time.Second,
+	}
+	checker := NewHTTPHealthzCheckerWithClient("foo", srv.URL, client, func(resp io.Reader) error {
 		return nil
 	})
 	var reporter health.Probes