From 7b483274cf44f5655d4a319fba651d46ceadc8c4 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Sun, 13 Oct 2024 15:20:16 +0530 Subject: [PATCH 1/6] added a Oci function to support generic image Signed-off-by: 7h3-3mp7y-m4n --- go.mod | 4 ++++ go.sum | 8 ++++++++ uriget/uriget.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/go.mod b/go.mod index 997217d..60e4ed1 100644 --- a/go.mod +++ b/go.mod @@ -13,5 +13,9 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sync v0.6.0 // indirect + oras.land/oras-go/v2 v2.5.0 // indirect ) diff --git a/go.sum b/go.sum index 44a92f4..5ba1d73 100644 --- a/go.sum +++ b/go.sum @@ -2,13 +2,21 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= diff --git a/uriget/uriget.go b/uriget/uriget.go index 4639d69..ccbf7ff 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -16,6 +16,10 @@ import ( "path/filepath" "strings" "time" + + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/registry/remote" ) // options is a struct holding fields that may need to have overrides in certain environments or during unit testing. @@ -110,6 +114,8 @@ func GetFile(ctx context.Context, rawUri string, optionFuncs ...Option) ([]byte, fallthrough case "git-https": return opts.getGit(ctx, u) + case "oci": + return opts.getOci(ctx, u) default: return nil, fmt.Errorf("unsupported scheme '%s'", u.Scheme) } @@ -233,3 +239,31 @@ func (o *options) getGit(ctx context.Context, u *url.URL) ([]byte, error) { o.logger.Printf("Read %d bytes from %s", len(buff), filepath.Join(td, subPath)) return buff, nil } + +func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { + parts := strings.Split(u.Host+u.Path, "/") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid OCI URL format") + } + registry := parts[0] + repo := strings.Join(parts[1:], "/") + tag := "latest" + if u.Fragment != "" { + tag = u.Fragment + } + store, err := oci.New(o.tempDir) + if err != nil { + return nil, fmt.Errorf("failed to create OCI layout store: %w", err) + } + repoUrl := fmt.Sprintf("%s/%s", registry, repo) + remoteRepo, err := remote.NewRepository(repoUrl) + if err != nil { + return nil, fmt.Errorf("failed to connect to remote repository: %w", err) + } + manifestDescriptor, err := oras.Copy(ctx, remoteRepo, tag, store, tag, oras.DefaultCopyOptions) + if err != nil { + return nil, fmt.Errorf("failed to pull OCI image: %w", err) + } + o.logger.Printf("Pulled OCI image: %s with manifest descriptor : %v", u.String(), manifestDescriptor.Digest) + return []byte(manifestDescriptor.Digest), nil +} From 95151cd3e9573f23ded588f6dd1d03f02d6554af Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Tue, 15 Oct 2024 14:58:53 +0530 Subject: [PATCH 2/6] added the unit test and updated the function for remote url Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 21 +++++++++++++++++++++ uriget/uriget.go | 15 +++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index ae490df..a7f0b38 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -3,8 +3,11 @@ package uriget import ( "context" "fmt" + "log" "net/http" "net/url" + "os" + "testing" ) func ExampleGetFile_local() { @@ -51,3 +54,21 @@ func ExampleWithHttpClient() { fmt.Println(err) // Output: failed to make get request: Get "https://example.com": no proxy } + +func TestGetOci(t *testing.T) { + logger := log.New(os.Stdout, "TEST: ", log.LstdFlags) + o := &options{ + tempDir: t.TempDir(), + logger: logger, + } + testUrl := "oci://localhost:5001/myimage:latest" + u, err := url.Parse(testUrl) + if err != nil { + t.Fatalf("failed to parse URL: %v", err) + } + + _, err = o.getOci(context.Background(), u) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } +} diff --git a/uriget/uriget.go b/uriget/uriget.go index ccbf7ff..41f98b1 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -246,10 +246,13 @@ func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { return nil, fmt.Errorf("invalid OCI URL format") } registry := parts[0] - repo := strings.Join(parts[1:], "/") + repo := strings.Join(parts[1:len(parts)-1], "/") tag := "latest" - if u.Fragment != "" { - tag = u.Fragment + lastPart := parts[len(parts)-1] + if strings.Contains(lastPart, ":") { + split := strings.Split(lastPart, ":") + repo = strings.Join(parts[1:len(parts)-1], "/") + "/" + split[0] + tag = split[1] } store, err := oci.New(o.tempDir) if err != nil { @@ -260,10 +263,14 @@ func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { if err != nil { return nil, fmt.Errorf("failed to connect to remote repository: %w", err) } + if strings.HasPrefix(repoUrl, "localhost:") || strings.HasPrefix(repoUrl, "127.0.0.1:") { + remoteRepo.PlainHTTP = true + } manifestDescriptor, err := oras.Copy(ctx, remoteRepo, tag, store, tag, oras.DefaultCopyOptions) if err != nil { return nil, fmt.Errorf("failed to pull OCI image: %w", err) } - o.logger.Printf("Pulled OCI image: %s with manifest descriptor : %v", u.String(), manifestDescriptor.Digest) + + o.logger.Printf("Pulled OCI image: %s with manifest descriptor: %v", u.String(), manifestDescriptor.Digest) return []byte(manifestDescriptor.Digest), nil } From f02507fecd1c51af852a82160a728cf970ff4df3 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Tue, 15 Oct 2024 19:40:28 +0530 Subject: [PATCH 3/6] Added the UnitTest with right image pull Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index a7f0b38..9bfb229 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -3,11 +3,8 @@ package uriget import ( "context" "fmt" - "log" "net/http" "net/url" - "os" - "testing" ) func ExampleGetFile_local() { @@ -54,21 +51,14 @@ func ExampleWithHttpClient() { fmt.Println(err) // Output: failed to make get request: Get "https://example.com": no proxy } - -func TestGetOci(t *testing.T) { - logger := log.New(os.Stdout, "TEST: ", log.LstdFlags) - o := &options{ - tempDir: t.TempDir(), - logger: logger, - } - testUrl := "oci://localhost:5001/myimage:latest" - u, err := url.Parse(testUrl) - if err != nil { - t.Fatalf("failed to parse URL: %v", err) - } - - _, err = o.getOci(context.Background(), u) +func ExampleWithOci() { + testUrl := "oci://ghcr.io/score-spec/score-compose:0.18.0" + buff, err := GetFile(context.Background(), testUrl) if err != nil { - t.Fatalf("expected no error, got: %v", err) + fmt.Println("failed to pull OCI image:", err) + return } + fmt.Println(len(buff) > 0, err) + // Output: + // true } From 666ca8921b61817b7255ca7dae2ec984bfad1680 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Tue, 15 Oct 2024 19:45:49 +0530 Subject: [PATCH 4/6] fix test function name Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index 9bfb229..6bff470 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -51,7 +51,7 @@ func ExampleWithHttpClient() { fmt.Println(err) // Output: failed to make get request: Get "https://example.com": no proxy } -func ExampleWithOci() { +func ExampleGetFile_Oci() { testUrl := "oci://ghcr.io/score-spec/score-compose:0.18.0" buff, err := GetFile(context.Background(), testUrl) if err != nil { From 0c8b008a35a834c244c4a34c9b3f72c2fd6d5e79 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Tue, 15 Oct 2024 19:51:19 +0530 Subject: [PATCH 5/6] fix example_test name convention Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index 6bff470..a655e25 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -51,7 +51,7 @@ func ExampleWithHttpClient() { fmt.Println(err) // Output: failed to make get request: Get "https://example.com": no proxy } -func ExampleGetFile_Oci() { +func ExampleGetFile_oci() { testUrl := "oci://ghcr.io/score-spec/score-compose:0.18.0" buff, err := GetFile(context.Background(), testUrl) if err != nil { From b417b1efae6768d99cbd90b44880ba7bcab68444 Mon Sep 17 00:00:00 2001 From: 7h3-3mp7y-m4n Date: Thu, 17 Oct 2024 23:06:03 +0530 Subject: [PATCH 6/6] Added the changes and rest of the testcases Signed-off-by: 7h3-3mp7y-m4n --- uriget/example_test.go | 28 ++++++++++++++++++++++++++-- uriget/uriget.go | 24 ++++++++---------------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/uriget/example_test.go b/uriget/example_test.go index a655e25..f4ad784 100644 --- a/uriget/example_test.go +++ b/uriget/example_test.go @@ -58,7 +58,31 @@ func ExampleGetFile_oci() { fmt.Println("failed to pull OCI image:", err) return } - fmt.Println(len(buff) > 0, err) + fmt.Println(len(buff) > 0) // Output: - // true + // true +} + +func ExampleGetFile_ociNoTag() { + testUrl := "oci://ghcr.io/score-spec/score-compose" + buff, err := GetFile(context.Background(), testUrl) + if err != nil { + fmt.Println("failed to pull OCI image:", err) + return + } + fmt.Println(len(buff) > 0) + // Output: + // true +} + +func ExampleGetFile_ociWithDigest() { + testUrl := "oci://ghcr.io/score-spec/score-compose@sha256:f3d8d5485a751cbdc91e073df1b6fbcde83f85a86ee3bc7d53e05b00452baedd" + buff, err := GetFile(context.Background(), testUrl) + if err != nil { + fmt.Println("failed to pull OCI image:", err) + return + } + fmt.Println(len(buff) > 0) + // Output: + // true } diff --git a/uriget/uriget.go b/uriget/uriget.go index 41f98b1..2d3a772 100644 --- a/uriget/uriget.go +++ b/uriget/uriget.go @@ -19,6 +19,7 @@ import ( "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/oci" + "oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry/remote" ) @@ -241,30 +242,21 @@ func (o *options) getGit(ctx context.Context, u *url.URL) ([]byte, error) { } func (o *options) getOci(ctx context.Context, u *url.URL) ([]byte, error) { - parts := strings.Split(u.Host+u.Path, "/") - if len(parts) < 2 { - return nil, fmt.Errorf("invalid OCI URL format") - } - registry := parts[0] - repo := strings.Join(parts[1:len(parts)-1], "/") - tag := "latest" - lastPart := parts[len(parts)-1] - if strings.Contains(lastPart, ":") { - split := strings.Split(lastPart, ":") - repo = strings.Join(parts[1:len(parts)-1], "/") + "/" + split[0] - tag = split[1] + ref, err := registry.ParseReference(u.Host + u.Path) + if err != nil { + return nil, fmt.Errorf("can't parse artifact URL into a valid reference: %w", err) } store, err := oci.New(o.tempDir) if err != nil { return nil, fmt.Errorf("failed to create OCI layout store: %w", err) } - repoUrl := fmt.Sprintf("%s/%s", registry, repo) - remoteRepo, err := remote.NewRepository(repoUrl) + remoteRepo, err := remote.NewRepository(ref.String()) if err != nil { return nil, fmt.Errorf("failed to connect to remote repository: %w", err) } - if strings.HasPrefix(repoUrl, "localhost:") || strings.HasPrefix(repoUrl, "127.0.0.1:") { - remoteRepo.PlainHTTP = true + tag := "latest" + if ref.Reference != "" { + tag = ref.Reference } manifestDescriptor, err := oras.Copy(ctx, remoteRepo, tag, store, tag, oras.DefaultCopyOptions) if err != nil {