-
Notifications
You must be signed in to change notification settings - Fork 7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: introducing oci artifact registry impl #2989
Changes from 4 commits
63b6c30
e7a480d
02d4279
3ae28f8
66552ff
aa8f92c
941e5a3
bc52571
9521ee0
ea77771
2d0618a
6b87750
2cc617a
1d48172
f026f30
3d793c4
0a27669
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
.hermit/ | ||
.vscode/* | ||
.registry/ | ||
!/.vscode/settings.json | ||
!/.vscode/launch.json | ||
.idea/* | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
package artefacts | ||
|
||
import ( | ||
"context" | ||
"encoding/hex" | ||
"fmt" | ||
"io" | ||
|
||
"oras.land/oras-go/v2" | ||
"oras.land/oras-go/v2/registry/remote" | ||
"oras.land/oras-go/v2/registry/remote/auth" | ||
"oras.land/oras-go/v2/registry/remote/retry" | ||
|
||
"github.com/TBD54566975/ftl/backend/controller/artefacts/internal/sql" | ||
"github.com/TBD54566975/ftl/backend/libdal" | ||
"github.com/TBD54566975/ftl/internal/model" | ||
"github.com/TBD54566975/ftl/internal/sha256" | ||
) | ||
|
||
type ContainerConfig struct { | ||
Registry string | ||
Repository string | ||
Username string | ||
Password string | ||
} | ||
|
||
type ContainerService struct { | ||
host string | ||
repository *remote.Repository | ||
// in the interim releases and artefacts will continue to be linked via the `deployment_artefacts` table | ||
Handle *libdal.Handle[ContainerService] | ||
db sql.Querier | ||
} | ||
|
||
func NewContainerService(c ContainerConfig, conn libdal.Connection) *ContainerService { | ||
repository, err := remote.NewRepository(fmt.Sprintf("%s/%s", c.Registry, c.Repository)) | ||
if err != nil { | ||
panic(fmt.Errorf("unable to connect to OCI repository: %w", err)) | ||
} | ||
|
||
client := &auth.Client{ | ||
Client: retry.DefaultClient, | ||
Cache: auth.NewCache(), | ||
Credential: auth.StaticCredential(c.Registry, auth.Credential{ | ||
Username: c.Username, | ||
Password: c.Password, | ||
}), | ||
} | ||
|
||
s := &ContainerService{ | ||
host: c.Registry, | ||
repository: repository, | ||
Handle: libdal.New(conn, func(h *libdal.Handle[ContainerService]) *ContainerService { | ||
svc := &ContainerService{ | ||
host: c.Registry, | ||
repository: repository, | ||
Handle: h, | ||
db: sql.New(h.Connection), | ||
} | ||
svc.repository.Client = client | ||
return svc | ||
}), | ||
} | ||
|
||
return s | ||
} | ||
|
||
func (s *ContainerService) GetDigestsKeys(ctx context.Context, digests []sha256.SHA256) (keys []ArtefactKey, missing []sha256.SHA256, err error) { | ||
return nil, nil, nil | ||
} | ||
|
||
func (s *ContainerService) Upload(ctx context.Context, artefact Artefact) (sha256.SHA256, error) { | ||
_, err := oras.PushBytes(ctx, s.repository, remoteModulePath(s.host, artefact.Digest), artefact.Content) | ||
if err != nil { | ||
return sha256.SHA256{}, fmt.Errorf("unable to upload artefact: %w", err) | ||
} | ||
return sha256.SHA256{}, nil | ||
} | ||
|
||
func (s *ContainerService) Download(ctx context.Context, digest sha256.SHA256) (io.ReadCloser, error) { | ||
_, stream, err := oras.Fetch(ctx, s.repository, remoteModulePath(s.host, digest), oras.DefaultFetchOptions) | ||
if err != nil { | ||
return nil, fmt.Errorf("unable to download artefact: %w", err) | ||
} | ||
return stream, nil | ||
} | ||
|
||
func (s *ContainerService) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) { | ||
return nil, nil | ||
} | ||
|
||
func (s *ContainerService) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error { | ||
return nil | ||
} | ||
|
||
func remoteModulePath(host string, digest sha256.SHA256) string { | ||
return fmt.Sprintf("%s/modules/%s:latest", host, hex.EncodeToString(digest[:])) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"crypto/sha256" | ||
"fmt" | ||
|
||
"github.com/google/uuid" | ||
|
||
"github.com/TBD54566975/ftl/backend/controller/artefacts" | ||
internalobservability "github.com/TBD54566975/ftl/internal/observability" | ||
) | ||
|
||
type releaseCmd struct { | ||
Describe releaseDescribeCmd `cmd:"" help:"Describes the specified release."` | ||
Publish releasePublishCmd `cmd:"" help:"Packages the project into a release and publishes it."` | ||
List releaseListCmd `cmd:"" help:"Lists all published releases."` | ||
} | ||
|
||
type releaseDescribeCmd struct { | ||
Digest string `arg:"" help:"Digest of the target release."` | ||
} | ||
|
||
func (d *releaseDescribeCmd) Run() error { | ||
return fmt.Errorf("release describe not implemented") | ||
} | ||
|
||
type releasePublishCmd struct { | ||
DSN string `help:"DAL DSN." default:"postgres://127.0.0.1:15432/ftl?sslmode=disable&user=postgres&password=secret" env:"FTL_CONTROLLER_DSN"` | ||
MaxOpenDBConnections int `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"` | ||
MaxIdleDBConnections int `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"` | ||
} | ||
|
||
func (d *releasePublishCmd) Run() error { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we are going to do 'releases' to a container registry we will need to push some kind of manifest with the contents of the release, as releases are multiple files. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. agreed, the transition to the OCI registries involves multiple steps, the first is to simply dump the payloads there instead of the database. from there it will be refined to take advantage of the container formats capabilities |
||
conn, err := internalobservability.OpenDBAndInstrument(d.DSN) | ||
if err != nil { | ||
return fmt.Errorf("failed to open DB connection: %w", err) | ||
} | ||
conn.SetMaxIdleConns(d.MaxIdleDBConnections) | ||
conn.SetMaxOpenConns(d.MaxOpenDBConnections) | ||
|
||
svc := artefacts.NewContainerService(artefacts.ContainerConfig{ | ||
Registry: "localhost:5000", | ||
Repository: "demo-repo", | ||
}, conn) | ||
content := uuid.New() | ||
contentBytes := content[:] | ||
_, err = svc.Upload(context.Background(), artefacts.Artefact{ | ||
Digest: sha256.Sum256(contentBytes), | ||
Metadata: artefacts.Metadata{ | ||
Path: fmt.Sprintf("random/%s", content), | ||
}, | ||
Content: contentBytes, | ||
}) | ||
if err != nil { | ||
return fmt.Errorf("failed upload artefact: %w", err) | ||
} | ||
return nil | ||
} | ||
|
||
type releaseListCmd struct { | ||
} | ||
|
||
func (d *releaseListCmd) Run() error { | ||
return fmt.Errorf("release list not implemented") | ||
} |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
FYI the typical pattern for config is to tag them with Kong tags, and embed them in the CLI so they can be automatically passed through from the CLI.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
updated