feat: introducing oci artifact registry impl

.gitignore

@@ -1,5 +1,6 @@
 .hermit/
 .vscode/*
+.registry/
 !/.vscode/settings.json
 !/.vscode/launch.json
 .idea/*

backend/controller/artefacts/dal_registry.go

@@ -76,7 +76,11 @@ func (s *Service) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCl
 }
 
 func (s *Service) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
-	rows, err := s.db.GetDeploymentArtefacts(ctx, releaseID)
+	return getDatabaseReleaseArtefacts(ctx, s.db, releaseID)
+}
+
+func getDatabaseReleaseArtefacts(ctx context.Context, db sql.Querier, releaseID int64) ([]ReleaseArtefact, error) {
+	rows, err := db.GetDeploymentArtefacts(ctx, releaseID)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get release artefacts: %w", libdal.TranslatePGError(err))
 	}

backend/controller/artefacts/oci_registry.go

@@ -0,0 +1,98 @@
+package artefacts
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/registry/remote"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/retry"
+
+	"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql"
+	"github.com/TBD54566975/ftl/backend/libdal"
+	"github.com/TBD54566975/ftl/internal/model"
+	"github.com/TBD54566975/ftl/internal/sha256"
+)
+
+type ContainerConfig struct {
+	Registry   string
+	Repository string
+	Username   string
+	Password   string
+}
+
+type ContainerService struct {
+	host       string
+	repository *remote.Repository
+	// in the interim releases and artefacts will continue to be linked via the `deployment_artefacts` table
+	Handle *libdal.Handle[ContainerService]
+	db     sql.Querier
+}
+
+func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerService {
+	repository, err := remote.NewRepository(fmt.Sprintf("%s/%s", c.Registry, c.Repository))
+	if err != nil {
+		panic(fmt.Errorf("unable to connect to OCI repository: %w", err))
+	}
+
+	client := &auth.Client{
+		Client: retry.DefaultClient,
+		Cache:  auth.NewCache(),
+		Credential: auth.StaticCredential(c.Registry, auth.Credential{
+			Username: c.Username,
+			Password: c.Password,
+		}),
+	}
+
+	s := &ContainerService{
+		host:       c.Registry,
+		repository: repository,
+		Handle: libdal.New(conn, func(h *libdal.Handle[ContainerService]) *ContainerService {
+			svc := &ContainerService{
+				host:       c.Registry,
+				repository: repository,
+				Handle:     h,
+				db:         sql.New(h.Connection),
+			}
+			svc.repository.Client = client
+			return svc
+		}),
+	}
+
+	return s
+}
+
+func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.SHA256) (keys []ArtefactKey, missing []sha256.SHA256, err error) {
+	return nil, nil, nil
+}
+
+func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha256.SHA256, error) {
+	_, err := oras.PushBytes(ctx, s.repository, remoteModulePath(s.host, artefact.Digest), artefact.Content)
+	if err != nil {
+		return sha256.SHA256{}, fmt.Errorf("unable to upload artefact: %w", err)
+	}
+	return sha256.SHA256{}, nil
+}
+
+func (s *ContainerService) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCloser, error) {
+	_, stream, err := oras.Fetch(ctx, s.repository, remoteModulePath(s.host, digest), oras.DefaultFetchOptions)
+	if err != nil {
+		return nil, fmt.Errorf("unable to download artefact: %w", err)
+	}
+	return stream, nil
+}
+
+func (s *ContainerService) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
+	return nil, nil
+}
+
+func (s *ContainerService) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error {
+	return nil
+}
+
+func remoteModulePath(host string, digest sha256.SHA256) string {
+	return fmt.Sprintf("%s/modules/%s:latest", host, hex.EncodeToString(digest[:]))
+}

docker-compose.yml

@@ -53,6 +53,12 @@ services:
     environment:
       SERVICES: secretsmanager
       DEBUG: 1
+  registry:
+    image: registry:2
+    ports:
+      -  "5000:5000"
+    volumes:
+      - ./.registry:/var/lib/registry
 
 volumes:
   grafana-storage: {}

frontend/cli/cmd_release.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/google/uuid"
+
+	"github.com/TBD54566975/ftl/backend/controller/artefacts"
+	internalobservability "github.com/TBD54566975/ftl/internal/observability"
+)
+
+type releaseCmd struct {
+	Describe releaseDescribeCmd `cmd:"" help:"Describes the specified release."`
+	Publish  releasePublishCmd  `cmd:"" help:"Packages the project into a release and publishes it."`
+	List     releaseListCmd     `cmd:"" help:"Lists all published releases."`
+}
+
+type releaseDescribeCmd struct {
+	Digest string `arg:"" help:"Digest of the target release."`
+}
+
+func (d *releaseDescribeCmd) Run() error {
+	return fmt.Errorf("release describe not implemented")
+}
+
+type releasePublishCmd struct {
+	DSN                  string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"`
+	MaxOpenDBConnections int    `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
+	MaxIdleDBConnections int    `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
+}
+
+func (d *releasePublishCmd) Run() error {
+	conn, err := internalobservability.OpenDBAndInstrument(d.DSN)
+	if err != nil {
+		return fmt.Errorf("failed to open DB connection: %w", err)
+	}
+	conn.SetMaxIdleConns(d.MaxIdleDBConnections)
+	conn.SetMaxOpenConns(d.MaxOpenDBConnections)
+
+	svc := artefacts.NewContainerService(artefacts.ContainerConfig{
+		Registry:   "localhost:5000",
+		Repository: "demo-repo",
+	}, conn)
+	content := uuid.New()
+	contentBytes := content[:]
+	_, err = svc.Upload(context.Background(), artefacts.Artefact{
+		Digest: sha256.Sum256(contentBytes),
+		Metadata: artefacts.Metadata{
+			Path: fmt.Sprintf("random/%s", content),
+		},
+		Content: contentBytes,
+	})
+	if err != nil {
+		return fmt.Errorf("failed upload artefact: %w", err)
+	}
+	return nil
+}
+
+type releaseListCmd struct {
+}
+
+func (d *releaseListCmd) Run() error {
+	return fmt.Errorf("release list not implemented")
+}

frontend/cli/main.go

@@ -54,6 +54,7 @@ type InteractiveCLI struct {
 	Secret   secretCmd   `cmd:"" help:"Manage secrets."`
 	Config   configCmd   `cmd:"" help:"Manage configuration."`
 	Pubsub   pubsubCmd   `cmd:"" help:"Manage pub/sub."`
+	Release  releaseCmd  `cmd:"" help:"Manage releases."`
 }
 
 type CLI struct {

go.mod

@@ -80,6 +80,7 @@ require (
 	k8s.io/apimachinery v0.31.1
 	k8s.io/client-go v0.31.1
 	modernc.org/sqlite v1.33.1
+	oras.land/oras-go/v2 v2.5.0
 )
 
 require (