[registry-facade] Add health probes to validate network access

components/registry-facade-api/go/config/config.go

@@ -11,10 +11,11 @@ import (
 
 // ServiceConfig configures this service
 type ServiceConfig struct {
-	Registry       Config `json:"registry"`
-	AuthCfg        string `json:"dockerAuth"`
-	PProfAddr      string `json:"pprofAddr"`
-	PrometheusAddr string `json:"prometheusAddr"`
+	Registry           Config `json:"registry"`
+	AuthCfg            string `json:"dockerAuth"`
+	PProfAddr          string `json:"pprofAddr"`
+	PrometheusAddr     string `json:"prometheusAddr"`
+	ReadinessProbeAddr string `json:"readinessProbeAddr"`
 }
 
 // GetConfig loads and validates the configuration

components/registry-facade/cmd/run.go

@@ -7,10 +7,11 @@ package cmd
 import (
 	"crypto/sha256"
 	"encoding/hex"
-	"github.com/gitpod-io/gitpod/registry-facade/api/config"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -20,6 +21,7 @@ import (
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/docker/cli/cli/config/configfile"
+	"github.com/heptiolabs/healthcheck"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -29,6 +31,7 @@ import (
 	common_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/pprof"
+	"github.com/gitpod-io/gitpod/registry-facade/api/config"
 	"github.com/gitpod-io/gitpod/registry-facade/pkg/registry"
 )
 
@@ -109,11 +112,39 @@ var runCmd = &cobra.Command{
 			return docker.NewResolver(resolverOpts)
 		}
 
+		if cfg.ReadinessProbeAddr != "" {
+			// use the first layer as source for the tests
+			staticLayerRef := cfg.Registry.StaticLayer[0].Ref
+			_, desc, err := resolverProvider().Resolve(context.Background(), staticLayerRef)
+			if err != nil {
+				log.WithError(err).Fatal("cannot resolve static layer reference")
+			}
+
+			staticLayerURL, err := url.Parse(desc.URLs[0])
+			if err != nil {
+				log.WithError(err).Fatalf("Invalid static layer found in registry-facade configuration: %v", staticLayerRef)
+			}
+
+			// Ensure we can resolve DNS queries, can access a layer of the StaticLayer image and the local Registry is up and running
+			health := healthcheck.NewHandler()
+			health.AddReadinessCheck("dns", dnsResolveCheck(staticLayerURL.Host, 1*time.Second))
+			health.AddReadinessCheck("registry", networkCheck(staticLayerURL.String()))
+			health.AddReadinessCheck("dns", healthcheck.DNSResolveCheck(staticLayerURL.Host, 1*time.Second))
+			health.AddReadinessCheck("registry", networkCheck(staticLayerURL.String()))
+
+			go func() {
+				if err := http.ListenAndServe(cfg.ReadinessProbeAddr, health); err != nil && err != http.ErrServerClosed {
+					log.WithError(err).Panic("error starting HTTP server")
+				}
+			}()
+		}
+
 		registryDoneChan := make(chan struct{})
 		reg, err := registry.NewRegistry(cfg.Registry, resolverProvider, prometheus.WrapRegistererWithPrefix("registry_", gpreg))
 		if err != nil {
 			log.WithError(err).Fatal("cannot create registry")
 		}
+
 		go watchConfig(configPath, reg)
 		go func() {
 			defer close(registryDoneChan)
@@ -143,7 +174,6 @@ func newDefaultTransport() *http.Transport {
 		IdleConnTimeout:       30 * time.Second,
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 5 * time.Second,
-		DisableKeepAlives:     true,
 	}
 }
 
@@ -220,3 +250,51 @@ func watchConfig(fn string, reg *registry.Registry) {
 		}
 	}
 }
+
+func networkCheck(url string) healthcheck.Check {
+	log.Infof("creating network check using URL %v", url)
+	return func() error {
+		client := http.Client{
+			Timeout: 1 * time.Second,
+			// never follow redirects
+			CheckRedirect: func(*http.Request, []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		}
+
+		resp, err := client.Get(url)
+		if err != nil {
+			log.Errorf("unexpected error checking URL %v: %v", url, err)
+			return err
+		}
+		resp.Body.Close()
+
+		if resp.StatusCode > 399 {
+			log.Errorf("unexpected status code checking URL %v (%v)", url, resp.StatusCode)
+			return fmt.Errorf("returned status %d", resp.StatusCode)
+		}
+
+		return nil
+	}
+}
+
+func dnsResolveCheck(host string, timeout time.Duration) func() error {
+	log.Infof("creating DNS check for host %v", host)
+	resolver := net.Resolver{}
+	return func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), timeout)
+		defer cancel()
+
+		addrs, err := resolver.LookupHost(ctx, host)
+		if err != nil {
+			log.Errorf("unexpected error resolving host %v: %v", host, err)
+			return err
+		}
+
+		if len(addrs) < 1 {
+			return fmt.Errorf("could not resolve host")
+		}
+
+		return nil
+	}
+}

components/registry-facade/go.mod

@@ -24,6 +24,8 @@ require (
 	google.golang.org/grpc v1.43.0
 )
 
+require github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb
+
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -50,6 +52,7 @@ require (
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
 )
 
 replace github.com/gitpod-io/gitpod/gitpod-protocol => ../gitpod-protocol/go // leeway