Add: Supports Grafana Cloud Logs

We need a simple and modern way to manage our logs so that when an issue
occurs, we can debug it easily. We now have a way to do this via
[Fluent Bit Loki Plugin](https://docs.fluentbit.io/manual/pipeline/outputs/loki). Reasons for going with fluent-bit:

* Very Very low memory footprint, this means we can run this service on
  a light machine and not have to worry about it, we can also share the
  host for main backend and fluent-bit.
* Very customizable - Fluent Bit has tons of plugins, which means we can
  redirect our logs to a different log management service, receive logs
  in various ways, filter log messages, etc.
* Support for multiple outputs - This is a strong feature of FluentBit.
  This lets us output our logs to Granfa Loki, Logzio, Datadog, StdOut, etc all
  at the same time.

Signed-off-by: jay-dee7 <[email protected]>

config/config.go

@@ -11,13 +11,14 @@ import (
 
 type (
 	RegistryConfig struct {
-		Environment     string       `mapstructure:"environment"`
 		AuthConfig      AuthConfig   `mapstructure:"auth_config"`
+		LogConfig       LogConfig    `mapstructure:"log_config"`
 		SkynetConfig    SkynetConfig `mapstructure:"skynet_config"`
-		Host            string       `mapstructure:"host"`
+		Environment     string       `mapstructure:"environment"`
 		DNSAddress      string       `mapstructure:"dns_address"`
 		SkynetPortalURL string       `mapstructure:"skynet_portal_url"`
 		SigningSecret   string       `mapstructure:"signing_secret"`
+		Host            string       `mapstructure:"host"`
 		Port            uint         `mapstructure:"port"`
 		Debug           bool         `mapstructure:"debug"`
 	}
@@ -31,6 +32,14 @@ type (
 		ApiKey          string `mapstructure:"api_key"`
 		CustomUserAgent string `mapstructure:"custom_user_agent"`
 	}
+
+	LogConfig struct {
+		Service    string `mapstructure:"service"`
+		Endpoint   string `mapstructure:"endpoint"`
+		AuthMethod string `mapstructure:"auth_method"`
+		Username   string `mapstructure:"username"`
+		Password   string `mapstructure:"password"`
+	}
 )
 
 func (r *RegistryConfig) Address() string {
@@ -60,6 +69,13 @@ func LoadFromENV() (*RegistryConfig, error) {
 		AuthConfig: AuthConfig{
 			SupportedServices: make(map[string]bool),
 		},
+		LogConfig: LogConfig{
+			Service:    viper.GetString("LOG_SERVICE_NAME"),
+			Endpoint:   viper.GetString("LOG_SERVICE_HOST"),
+			AuthMethod: viper.GetString("LOG_SERVICE_AUTH_KIND"),
+			Username:   viper.GetString("LOG_SERVICE_USER"),
+			Password:   viper.GetString("LOG_SERVICE_PASSWORD"),
+		},
 	}
 
 	for _, service := range strings.Split(viper.GetString("SUPPORTED_SERVICES"), ",") {
@@ -82,11 +98,18 @@ func LoadFromENV() (*RegistryConfig, error) {
 
 func (r *RegistryConfig) Endpoint() string {
 	switch r.Environment {
-	case "dev", "devel", "development", "local":
+	case Dev, Local:
 		return fmt.Sprintf("http://%s:%d", r.Host, r.Port)
-	case "stage", "production":
+	case Prod, Stage:
 		return fmt.Sprintf("https://%s", r.DNSAddress)
 	default:
 		return fmt.Sprintf("http://%s:%d", r.Host, r.Port)
 	}
 }
+
+const (
+	Prod  = "production"
+	Stage = "stage"
+	Dev   = "development"
+	Local = "local"
+)

env-vars.example

@@ -6,3 +6,6 @@ OPEN_REGISTRY_SKYNET_PORTAL_URL=https://siasky.net
 OPEN_REGISTRY_SIGNING_SECRET="3tYnaKp@^%hbQA%J&x3cX!r2#mK%EBfAbTvPMv5CU2DP7bAoQGnUfT2&dW"
 OPEN_REGISTRY_SUPPORTED_SERVICES=github,token
 OPEN_REGISTRY_ENVIRONMENT=local
+OPEN_REGISTRY_LOG_SERVICE_NAME=grafana-loki
+OPEN_REGISTRY_LOG_SERVICE_HOST=http://0.0.0.0:9880/app.log
+OPEN_REGISTRY_LOG_SERVICE_AUTH_KIND=basic

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/spf13/viper v1.8.1
 	github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+    github.com/google/uuid v1.3.0
 )
 
 require (

go.sum

@@ -229,6 +229,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=

main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"log"
 	"os"
 
 	"github.com/containerish/OpenRegistry/auth"
@@ -11,6 +10,7 @@ import (
 	"github.com/containerish/OpenRegistry/router"
 	"github.com/containerish/OpenRegistry/skynet"
 	"github.com/containerish/OpenRegistry/telemetry"
+	fluentbit "github.com/containerish/OpenRegistry/telemetry/fluent-bit"
 	"github.com/fatih/color"
 	"github.com/labstack/echo/v4"
 )
@@ -34,13 +34,19 @@ func main() {
 	authSvc := auth.New(localCache, cfg)
 	skynetClient := skynet.NewClient(cfg)
 
-	l := telemetry.SetupLogger()
-	reg, err := registry.NewRegistry(skynetClient, l, localCache, e.Logger)
+	log := telemetry.SetupLogger()
+	fluentBitCollector, err := fluentbit.New(cfg)
+	if err != nil {
+		color.Red("error initializing fluentbit collector: %s\n", err)
+		os.Exit(1)
+	}
+
+	reg, err := registry.NewRegistry(skynetClient, log, localCache, e.Logger, fluentBitCollector)
 	if err != nil {
 		e.Logger.Errorf("error creating new container registry: %s", err)
 		return
 	}
 
 	router.Register(e, reg, authSvc, localCache)
-	log.Println(e.Start(cfg.Address()))
+	log.Fatal().Msgf("error starting server: %s\n", e.Start(cfg.Address()))
 }

registry/v2/blobs.go

@@ -37,12 +37,14 @@ func (b *blobs) HEAD(ctx echo.Context) error {
 			"skynet": "skynet link not found",
 		}
 		errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), details)
+		b.fluentbit.Send(errMsg)
 		return ctx.JSONBlob(http.StatusNotFound, errMsg)
 	}
 
 	size, ok := b.registry.skynet.Metadata(layerRef.Skylink)
 	if !ok {
 		errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil)
+		b.fluentbit.Send(errMsg)
 		return ctx.JSONBlob(http.StatusNotFound, errMsg)
 	}
 
@@ -63,6 +65,7 @@ func (b *blobs) UploadBlob(ctx echo.Context) error {
 				"stream upload after first write are not allowed",
 				nil,
 			)
+			b.fluentbit.Send(errMsg)
 			return ctx.JSONBlob(http.StatusBadRequest, errMsg)
 		}
 
@@ -85,11 +88,13 @@ func (b *blobs) UploadBlob(ctx echo.Context) error {
 			"contentRange": contentRange,
 		}
 		errMsg := b.errorResponse(RegistryErrorCodeBlobUploadUnknown, err.Error(), details)
+		b.fluentbit.Send(errMsg)
 		return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg)
 	}
 
 	if start != len(b.uploads[uuid]) {
 		errMsg := b.errorResponse(RegistryErrorCodeBlobUploadUnknown, "content range mismatch", nil)
+		b.fluentbit.Send(errMsg)
 		return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg)
 	}
 
@@ -101,6 +106,7 @@ func (b *blobs) UploadBlob(ctx echo.Context) error {
 			"error while creating new buffer from existing blobs",
 			nil,
 		)
+		b.fluentbit.Send(errMsg)
 		return ctx.JSONBlob(http.StatusInternalServerError, errMsg)
 	} // 10
 	ctx.Request().Body.Close()