From 190aae971fad0df9775bf2ab30ad1301bdf21a53 Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Wed, 7 Sep 2022 17:29:35 +0800 Subject: [PATCH 01/11] feat: enhance manifest fetch Signed-off-by: Haoliang Yue --- cmd/oras/manifest/cmd.go | 2 +- cmd/oras/manifest/fetch.go | 103 +++++++++++++++++++------- internal/cas/fetch.go | 71 ------------------ internal/cas/fetch_test.go | 145 ------------------------------------- 4 files changed, 77 insertions(+), 244 deletions(-) delete mode 100644 internal/cas/fetch.go delete mode 100644 internal/cas/fetch_test.go diff --git a/cmd/oras/manifest/cmd.go b/cmd/oras/manifest/cmd.go index 24b14488a..e3d6ab3b6 100644 --- a/cmd/oras/manifest/cmd.go +++ b/cmd/oras/manifest/cmd.go @@ -21,7 +21,7 @@ import ( func Cmd() *cobra.Command { cmd := &cobra.Command{ - Use: "manifest [fetch]", + Use: "manifest [command]", Short: "[Preview] Manifest operations", } diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index a50cbdcf5..315922b57 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -16,26 +16,29 @@ limitations under the License. package manifest import ( - "bytes" "encoding/json" - "fmt" + "errors" "os" "github.com/spf13/cobra" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/cas" + "oras.land/oras/internal/cache" ) type fetchOptions struct { option.Common + option.Descriptor option.Remote option.Platform + option.Pretty - targetRef string - pretty bool - mediaTypes []string - fetchDescriptor bool + cacheRoot string + mediaTypes []string + outputPath string + targetRef string } func fetchCmd() *cobra.Command { @@ -63,6 +66,10 @@ Example - Fetch manifest with prettified json result: `, Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { + if opts.outputPath == "-" && opts.OutputDescriptor { + return errors.New("`--output -` cannot be used with `--descriptor` at the same time") + } + return opts.ReadPassword() }, Aliases: []string{"get"}, @@ -72,46 +79,88 @@ Example - Fetch manifest with prettified json result: }, } - cmd.Flags().BoolVarP(&opts.pretty, "pretty", "", false, "output prettified manifest") - cmd.Flags().BoolVarP(&opts.fetchDescriptor, "descriptor", "", false, "fetch a descriptor of the manifest") cmd.Flags().StringSliceVarP(&opts.mediaTypes, "media-type", "", nil, "accepted media types") + cmd.Flags().StringVarP(&opts.outputPath, "output", "o", "", "output file path") option.ApplyFlags(&opts, cmd.Flags()) return cmd } func fetchManifest(opts fetchOptions) error { ctx, _ := opts.SetLoggerLevel() - targetPlatform, err := opts.Parse() - if err != nil { - return err - } + repo, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { return err } + if repo.Reference.Reference == "" { return oerrors.NewErrInvalidReference(repo.Reference) } repo.ManifestMediaTypes = opts.mediaTypes - // Fetch and output - var content []byte - if opts.fetchDescriptor { - content, err = cas.FetchDescriptor(ctx, repo, opts.targetRef, targetPlatform) - } else { - content, err = cas.FetchManifest(ctx, repo, opts.targetRef, targetPlatform) + targetPlatform, err := opts.Parse() + if err != nil { + return err + } + + var src oras.ReadOnlyTarget = repo.Manifests() + if opts.cacheRoot != "" { + ociStore, err := oci.New(opts.cacheRoot) + if err != nil { + return err + } + src = cache.New(src, ociStore) } + + // fetch manifest + desc, content, err := oras.FetchBytes(ctx, src, opts.targetRef, oras.FetchBytesOptions{ + FetchOptions: oras.FetchOptions{ + ResolveOptions: oras.ResolveOptions{ + TargetPlatform: targetPlatform, + }, + }, + MaxBytes: 0, + }) if err != nil { return err } - if opts.pretty { - buf := bytes.NewBuffer(nil) - if err = json.Indent(buf, content, "", " "); err != nil { - return fmt.Errorf("failed to prettify: %w", err) + + // output manifest's descriptor if `--descriptor` is used + if opts.OutputDescriptor { + descBytes, err := json.Marshal(desc) + if err != nil { + return err + } + err = opts.Output(os.Stdout, descBytes) + if err != nil { + return err + } + } + + // save manifest content into the local file if the output path is provided + if opts.outputPath != "" { + file, err := os.OpenFile(opts.outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer func() { + if closeErr := file.Close(); err == nil { + err = closeErr + } + }() + + _, err = file.Write(content) + if err != nil { + return err + } + } + + if opts.outputPath == "-" || (opts.outputPath == "" && !opts.OutputDescriptor) { + err = opts.Output(os.Stdout, content) + if err != nil { + return err } - buf.WriteByte('\n') - content = buf.Bytes() } - _, err = os.Stdout.Write(content) - return err + + return nil } diff --git a/internal/cas/fetch.go b/internal/cas/fetch.go deleted file mode 100644 index 2763157d3..000000000 --- a/internal/cas/fetch.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cas - -import ( - "context" - "encoding/json" - - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/content/memory" - "oras.land/oras-go/v2/registry" - "oras.land/oras/internal/cache" -) - -// FetchDescriptor fetches a minimal descriptor of reference from target. -// If platform flag not empty, will fetch the specified platform. -func FetchDescriptor(ctx context.Context, target oras.ReadOnlyTarget, reference string, p *ocispec.Platform) ([]byte, error) { - desc, err := oras.Resolve(ctx, target, reference, oras.ResolveOptions{TargetPlatform: p}) - if err != nil { - return nil, err - } - return json.Marshal(ocispec.Descriptor{ - MediaType: desc.MediaType, - Digest: desc.Digest, - Size: desc.Size, - }) -} - -// FetchManifest fetches the manifest content of reference from target. -// If platform flag not empty, will fetch the specified platform. -func FetchManifest(ctx context.Context, target oras.ReadOnlyTarget, reference string, p *ocispec.Platform) ([]byte, error) { - // TODO: improve implementation once oras-go#102 is resolved - if p == nil { - if rf, ok := target.(registry.ReferenceFetcher); ok { - desc, rc, err := rf.FetchReference(ctx, reference) - if err != nil { - return nil, err - } - defer rc.Close() - return content.ReadAll(rc, desc) - } - } - target = cache.New(target, memory.New()) - desc, err := oras.Resolve(ctx, target, reference, oras.ResolveOptions{ - TargetPlatform: p, - }) - if err != nil { - return nil, err - } - rc, err := target.Fetch(ctx, desc) - if err != nil { - return nil, err - } - defer rc.Close() - return content.ReadAll(rc, desc) -} diff --git a/internal/cas/fetch_test.go b/internal/cas/fetch_test.go deleted file mode 100644 index aaa50b3e8..000000000 --- a/internal/cas/fetch_test.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cas_test - -import ( - "bytes" - "context" - "errors" - "testing" - - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/errdef" - "oras.land/oras/internal/cas" - "oras.land/oras/internal/mock" -) - -const ( - index = `{"manifests":[{"digest":"sha256:baf0239e48ff4c47ebac3ba02b5cf1506b69cd5a0c0d0c825a53ba65976fb942","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"amd64","os":"linux"},"size":11},{"digest":"sha256:27cb13102d774dc36e0bc93f528db7e4f004a6e9636cb6926b1e389668535309","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v5"},"size":12}]}` - amd64 = "linux/amd64" - armv5 = "linux/arm/v5" - armv7 = "linux/arm/v7" - - indexDesc = `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:bdcc003fa2d7882789773fe5fee506ef370dce5ce7988fd420587f144fc700db","size":452}` - armv5Desc = `{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:27cb13102d774dc36e0bc93f528db7e4f004a6e9636cb6926b1e389668535309","size":12}` - amd64Desc = `{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","digest":"sha256:baf0239e48ff4c47ebac3ba02b5cf1506b69cd5a0c0d0c825a53ba65976fb942","size":11}` - badType = "application/a.not.supported.manifest.v2+jso" - badDesc = `{"mediaType":"application/a.not.supported.manifest.v2+json","digest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","size":0}` -) - -var repo = mock.New().WithFetch().WithFetchReference().WithResolve() - -func TestPlatform_FetchManifest_indexAndPlatform(t *testing.T) { - repo.Remount([]mock.Blob{ - {Content: index, MediaType: ocispec.MediaTypeImageIndex, Tag: ""}, - {Content: amd64, MediaType: ocispec.MediaTypeImageManifest, Tag: ""}, - {Content: armv5, MediaType: ocispec.MediaTypeImageManifest, Tag: ""}}) - - // Get index manifest - indexBytes := []byte(index) - got, err := cas.FetchManifest(context.Background(), repo, digest.FromBytes(indexBytes).String(), nil) - if err != nil || !bytes.Equal(got, indexBytes) { - t.Fatal(err) - } - - // Get manifest for specific platform - want := []byte(amd64) - got, err = cas.FetchManifest(context.Background(), repo, digest.FromBytes(indexBytes).String(), &ocispec.Platform{OS: "linux", Architecture: "amd64"}) - if err != nil || !bytes.Equal(got, want) { - t.Fatal(err) - } - - want = []byte(armv5) - got, err = cas.FetchManifest(context.Background(), repo, digest.FromBytes(indexBytes).String(), &ocispec.Platform{OS: "linux", Architecture: "arm", Variant: "v5"}) - if err != nil || !bytes.Equal(got, want) { - t.Fatal(err) - } -} - -func TestPlatform_FetchDescriptor_indexAndPlatform(t *testing.T) { - var indexTag = "multi-platform" - repo.Remount([]mock.Blob{ - {Content: index, MediaType: ocispec.MediaTypeImageIndex, Tag: indexTag}, - {Content: amd64, MediaType: ocispec.MediaTypeImageManifest, Tag: ""}, - {Content: armv5, MediaType: ocispec.MediaTypeImageManifest, Tag: ""}}) - - // Get index manifest - indexBytes := []byte(index) - got, err := cas.FetchDescriptor(context.Background(), repo, digest.FromBytes(indexBytes).String(), nil) - if err != nil || !bytes.Equal(got, []byte(indexDesc)) { - t.Fatal(err) - } - - // Get manifest for specific platform - want := []byte(amd64Desc) - got, err = cas.FetchDescriptor(context.Background(), repo, indexTag, &ocispec.Platform{OS: "linux", Architecture: "amd64"}) - if err != nil || !bytes.Equal(got, want) { - t.Fatal(err) - } - got, err = cas.FetchDescriptor(context.Background(), repo, indexTag, &ocispec.Platform{OS: "linux", Architecture: "arm", Variant: "v5"}) - if err != nil || !bytes.Equal(got, []byte(armv5Desc)) { - t.Fatal(err) - } -} - -func TestPlatform_FetchManifest_errNotMulti(t *testing.T) { - repo.Remount([]mock.Blob{{Content: "", MediaType: badType, Tag: badDesc}}) - - // Unknow media type - _, err := cas.FetchManifest(context.Background(), repo, digest.FromBytes([]byte("")).String(), &ocispec.Platform{OS: "linux", Architecture: "amd64"}) - if !errors.Is(err, errdef.ErrUnsupported) { - t.Fatalf("Expecting error: %v, got: %v", errdef.ErrUnsupported, err) - } -} -func TestPlatform_FetchManifest_errNoMatch(t *testing.T) { - // No matched platform found - repo.Remount([]mock.Blob{{Content: index, MediaType: ocispec.MediaTypeImageIndex, Tag: ""}}) - _, err := cas.FetchManifest( - context.Background(), - repo, - digest.FromBytes([]byte(index)).String(), - &ocispec.Platform{OS: "linux", Architecture: "arm", Variant: "v7"}) - if !errors.Is(err, errdef.ErrNotFound) { - t.Fatalf("Expecting error: %v, got: %v", errdef.ErrNotFound, err) - } -} - -func TestPlatform_FetchDescriptor_miscErr(t *testing.T) { - // Should throw err when repo is nil - repo.Remount(nil) - ret, err := cas.FetchDescriptor(context.Background(), repo, "invalid-RefERENCE", nil) - if err == nil { - t.Fatalf("Should fail oras.Resolve, unexpected return value: %v", ret) - } - -} - -func TestPlatform_FetchManifest_miscErr(t *testing.T) { - // Should throw err when repo is empty - repo.Remount(nil) - ret, err := cas.FetchManifest(context.Background(), repo, "mocked-reference", nil) - if err == nil { - t.Fatalf("Should fail oras.Resolve, unexpected return value: %v", ret) - } - // Should throw err when resolve succeeds but fetch reference fails - tmpRepo := mock.New().WithResolve() - tmpRepo.Remount([]mock.Blob{{Content: amd64, MediaType: ocispec.MediaTypeImageManifest, Tag: ""}}) - ret, err = cas.FetchManifest(context.Background(), tmpRepo, digest.FromBytes([]byte(amd64)).String(), nil) - if err == nil { - t.Fatalf("Should fail oras.Fetch, unexpected return value: %v", ret) - } -} From 7e265c95cec6301294a61c5acd13f7a138bb3f6a Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Fri, 9 Sep 2022 15:20:31 +0800 Subject: [PATCH 02/11] improve only fetch descriptor Signed-off-by: Haoliang Yue --- cmd/oras/manifest/fetch.go | 48 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index 315922b57..31affd2c7 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -20,6 +20,7 @@ import ( "errors" "os" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/oci" @@ -47,6 +48,7 @@ func fetchCmd() *cobra.Command { Use: "fetch [flags] ", Short: "[Preview] Fetch manifest of the target artifact", Long: `[Preview] Fetch manifest of the target artifact + ** This command is in preview and under development. ** Example - Fetch raw manifest: @@ -85,7 +87,7 @@ Example - Fetch manifest with prettified json result: return cmd } -func fetchManifest(opts fetchOptions) error { +func fetchManifest(opts fetchOptions) (fetchErr error) { ctx, _ := opts.SetLoggerLevel() repo, err := opts.NewRepository(opts.targetRef, opts.Common) @@ -112,17 +114,27 @@ func fetchManifest(opts fetchOptions) error { src = cache.New(src, ociStore) } - // fetch manifest - desc, content, err := oras.FetchBytes(ctx, src, opts.targetRef, oras.FetchBytesOptions{ - FetchOptions: oras.FetchOptions{ - ResolveOptions: oras.ResolveOptions{ - TargetPlatform: targetPlatform, + var desc ocispec.Descriptor + var content []byte + if opts.OutputDescriptor && opts.outputPath == "" { + // fetch manifest descriptor only + desc, err = oras.Resolve(ctx, src, opts.targetRef, oras.DefaultResolveOptions) + if err != nil { + return err + } + } else { + // fetch manifest content + desc, content, err = oras.FetchBytes(ctx, src, opts.targetRef, oras.FetchBytesOptions{ + FetchOptions: oras.FetchOptions{ + ResolveOptions: oras.ResolveOptions{ + TargetPlatform: targetPlatform, + }, }, - }, - MaxBytes: 0, - }) - if err != nil { - return err + MaxBytes: 0, + }) + if err != nil { + return err + } } // output manifest's descriptor if `--descriptor` is used @@ -138,24 +150,24 @@ func fetchManifest(opts fetchOptions) error { } // save manifest content into the local file if the output path is provided - if opts.outputPath != "" { - file, err := os.OpenFile(opts.outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if opts.outputPath != "" && opts.outputPath != "-" { + file, err := os.Create(opts.outputPath) if err != nil { return err } defer func() { - if closeErr := file.Close(); err == nil { - err = closeErr + if err := file.Close(); fetchErr == nil { + fetchErr = err } }() - _, err = file.Write(content) - if err != nil { + if _, err = file.Write(content); err != nil { return err } } - if opts.outputPath == "-" || (opts.outputPath == "" && !opts.OutputDescriptor) { + // outputs manifest content + if (opts.outputPath == "" && !opts.OutputDescriptor) || opts.outputPath == "-" { err = opts.Output(os.Stdout, content) if err != nil { return err From 58ab6b626de14b0d829d2bcfc16505627dcd87ce Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Fri, 9 Sep 2022 15:44:43 +0800 Subject: [PATCH 03/11] fix missing cacheRoot Signed-off-by: Haoliang Yue --- cmd/oras/manifest/fetch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index 31affd2c7..8964f55d6 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -72,6 +72,7 @@ Example - Fetch manifest with prettified json result: return errors.New("`--output -` cannot be used with `--descriptor` at the same time") } + opts.cacheRoot = os.Getenv("ORAS_CACHE") return opts.ReadPassword() }, Aliases: []string{"get"}, From 6f4f8c4a00fe1b112088298eda9b554d00159048 Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Tue, 13 Sep 2022 17:25:38 +0800 Subject: [PATCH 04/11] change the if else logic Signed-off-by: Haoliang Yue --- cmd/oras/manifest/fetch.go | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index 8964f55d6..f6f8a1f8c 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -136,22 +136,13 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { if err != nil { return err } - } - // output manifest's descriptor if `--descriptor` is used - if opts.OutputDescriptor { - descBytes, err := json.Marshal(desc) - if err != nil { - return err - } - err = opts.Output(os.Stdout, descBytes) - if err != nil { - return err + // outputs manifest content + if opts.outputPath == "" || opts.outputPath == "-" { + return opts.Output(os.Stdout, content) } - } - // save manifest content into the local file if the output path is provided - if opts.outputPath != "" && opts.outputPath != "-" { + // save manifest content into the local file if the output path is provided file, err := os.Create(opts.outputPath) if err != nil { return err @@ -167,9 +158,13 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { } } - // outputs manifest content - if (opts.outputPath == "" && !opts.OutputDescriptor) || opts.outputPath == "-" { - err = opts.Output(os.Stdout, content) + // output manifest's descriptor if `--descriptor` is used + if opts.OutputDescriptor { + descBytes, err := json.Marshal(desc) + if err != nil { + return err + } + err = opts.Output(os.Stdout, descBytes) if err != nil { return err } From 369e9889f6bbb3a0567367cda18016c0e012a5cb Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Thu, 15 Sep 2022 14:12:59 +0800 Subject: [PATCH 05/11] temporarily save changes Signed-off-by: Haoliang Yue --- cmd/oras/internal/option/cache.go | 41 ++++++++++++++++++++++++++ cmd/oras/internal/option/cache_test.go | 22 ++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 cmd/oras/internal/option/cache.go create mode 100644 cmd/oras/internal/option/cache_test.go diff --git a/cmd/oras/internal/option/cache.go b/cmd/oras/internal/option/cache.go new file mode 100644 index 000000000..d80f70bc2 --- /dev/null +++ b/cmd/oras/internal/option/cache.go @@ -0,0 +1,41 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package option + +import ( + "os" + + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras/internal/cache" +) + +type Cache struct { + CacheRoot string +} + +// CachedTarget gets the target storage with caching if cache root is specified. +func (opts *Cache) CachedTarget(src oras.ReadOnlyTarget) (oras.ReadOnlyTarget, error) { + opts.CacheRoot = os.Getenv("ORAS_CACHE") + if opts.CacheRoot != "" { + ociStore, err := oci.New(opts.CacheRoot) + if err != nil { + return nil, err + } + return cache.New(src, ociStore), nil + } + return src, nil +} diff --git a/cmd/oras/internal/option/cache_test.go b/cmd/oras/internal/option/cache_test.go new file mode 100644 index 000000000..da825f3e0 --- /dev/null +++ b/cmd/oras/internal/option/cache_test.go @@ -0,0 +1,22 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package option + +import "testing" + +func TestCache_CachedTarget(t *testing.T) { + +} From 424c05bcee1572a37c9c69cf11e6d214d218ef6f Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Thu, 15 Sep 2022 17:31:28 +0800 Subject: [PATCH 06/11] refactor cache to option package and make use of os.WriteFile Signed-off-by: Haoliang Yue --- cmd/oras/internal/option/cache_test.go | 42 ++++++++++++++++++- .../internal/option/nonexistent/oci-layout | 1 + cmd/oras/manifest/fetch.go | 12 +----- 3 files changed, 43 insertions(+), 12 deletions(-) create mode 100644 cmd/oras/internal/option/nonexistent/oci-layout diff --git a/cmd/oras/internal/option/cache_test.go b/cmd/oras/internal/option/cache_test.go index da825f3e0..a910cc8eb 100644 --- a/cmd/oras/internal/option/cache_test.go +++ b/cmd/oras/internal/option/cache_test.go @@ -15,8 +15,48 @@ limitations under the License. package option -import "testing" +import ( + "os" + "reflect" + "testing" + + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/memory" + "oras.land/oras-go/v2/content/oci" + "oras.land/oras/internal/cache" +) + +var mockTarget oras.ReadOnlyTarget = memory.New() func TestCache_CachedTarget(t *testing.T) { + tempDir := t.TempDir() + os.Setenv("ORAS_CACHE", tempDir) + opts := Cache{} + + ociStore, err := oci.New(tempDir) + if err != nil { + t.Fatal("error calling oci.New(), error =", err) + } + want := cache.New(mockTarget, ociStore) + + got, err := opts.CachedTarget(mockTarget) + if err != nil { + t.Fatal("Cache.CachedTarget() error=", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Cache.CachedTarget() got %v, want %v", got, want) + } +} + +func TestCache_CachedTarget_emptyRoot(t *testing.T) { + os.Setenv("ORAS_CACHE", "") + opts := Cache{} + got, err := opts.CachedTarget(mockTarget) + if err != nil { + t.Fatal("Cache.CachedTarget() error=", err) + } + if !reflect.DeepEqual(got, mockTarget) { + t.Fatalf("Cache.CachedTarget() got %v, want %v", got, mockTarget) + } } diff --git a/cmd/oras/internal/option/nonexistent/oci-layout b/cmd/oras/internal/option/nonexistent/oci-layout new file mode 100644 index 000000000..1343d370f --- /dev/null +++ b/cmd/oras/internal/option/nonexistent/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion":"1.0.0"} \ No newline at end of file diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index f6f8a1f8c..27718f124 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -143,17 +143,7 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { } // save manifest content into the local file if the output path is provided - file, err := os.Create(opts.outputPath) - if err != nil { - return err - } - defer func() { - if err := file.Close(); fetchErr == nil { - fetchErr = err - } - }() - - if _, err = file.Write(content); err != nil { + if err = os.WriteFile(opts.outputPath, content, 0666); err != nil { return err } } From 672c3571794430e99bc72d5e690a9cefe748726c Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Thu, 15 Sep 2022 17:34:38 +0800 Subject: [PATCH 07/11] delete the folder generated by mistake Signed-off-by: Haoliang Yue --- cmd/oras/internal/option/nonexistent/oci-layout | 1 - 1 file changed, 1 deletion(-) delete mode 100644 cmd/oras/internal/option/nonexistent/oci-layout diff --git a/cmd/oras/internal/option/nonexistent/oci-layout b/cmd/oras/internal/option/nonexistent/oci-layout deleted file mode 100644 index 1343d370f..000000000 --- a/cmd/oras/internal/option/nonexistent/oci-layout +++ /dev/null @@ -1 +0,0 @@ -{"imageLayoutVersion":"1.0.0"} \ No newline at end of file From 5287bf15fce97737e68ca3cff675320facb85509 Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Thu, 15 Sep 2022 17:46:31 +0800 Subject: [PATCH 08/11] change pull and blob fetch to use option.Cache Signed-off-by: Haoliang Yue --- cmd/oras/blob/fetch.go | 15 ++++----------- cmd/oras/manifest/fetch.go | 28 ++++++++-------------------- cmd/oras/pull.go | 16 ++++------------ 3 files changed, 16 insertions(+), 43 deletions(-) diff --git a/cmd/oras/blob/fetch.go b/cmd/oras/blob/fetch.go index 22aa4d2f7..1c120afc7 100644 --- a/cmd/oras/blob/fetch.go +++ b/cmd/oras/blob/fetch.go @@ -24,18 +24,16 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/oci" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/cache" ) type fetchBlobOptions struct { + option.Cache option.Common option.Descriptor option.Pretty option.Remote - cacheRoot string outputPath string targetRef string } @@ -74,7 +72,6 @@ Example - Fetch blob from the insecure registry: return errors.New("`--output -` cannot be used with `--descriptor` at the same time") } - opts.cacheRoot = os.Getenv("ORAS_CACHE") return opts.ReadPassword() }, Aliases: []string{"get"}, @@ -101,13 +98,9 @@ func fetchBlob(opts fetchBlobOptions) (fetchErr error) { return fmt.Errorf("%s: blob reference must be of the form ", opts.targetRef) } - var src oras.ReadOnlyTarget = repo.Blobs() - if opts.cacheRoot != "" { - ociStore, err := oci.New(opts.cacheRoot) - if err != nil { - return err - } - src = cache.New(src, ociStore) + src, err := opts.CachedTarget(repo.Blobs()) + if err != nil { + return err } var desc ocispec.Descriptor diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index 27718f124..6a21cdd89 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -23,20 +23,18 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/oci" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/cache" ) type fetchOptions struct { + option.Cache option.Common option.Descriptor option.Remote option.Platform option.Pretty - cacheRoot string mediaTypes []string outputPath string targetRef string @@ -72,7 +70,6 @@ Example - Fetch manifest with prettified json result: return errors.New("`--output -` cannot be used with `--descriptor` at the same time") } - opts.cacheRoot = os.Getenv("ORAS_CACHE") return opts.ReadPassword() }, Aliases: []string{"get"}, @@ -106,33 +103,24 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { return err } - var src oras.ReadOnlyTarget = repo.Manifests() - if opts.cacheRoot != "" { - ociStore, err := oci.New(opts.cacheRoot) - if err != nil { - return err - } - src = cache.New(src, ociStore) + manifests, err := opts.CachedTarget(repo.Manifests()) + if err != nil { + return err } var desc ocispec.Descriptor var content []byte if opts.OutputDescriptor && opts.outputPath == "" { // fetch manifest descriptor only - desc, err = oras.Resolve(ctx, src, opts.targetRef, oras.DefaultResolveOptions) + desc, err = oras.Resolve(ctx, manifests, opts.targetRef, oras.DefaultResolveOptions) if err != nil { return err } } else { // fetch manifest content - desc, content, err = oras.FetchBytes(ctx, src, opts.targetRef, oras.FetchBytesOptions{ - FetchOptions: oras.FetchOptions{ - ResolveOptions: oras.ResolveOptions{ - TargetPlatform: targetPlatform, - }, - }, - MaxBytes: 0, - }) + fetchOpts := oras.DefaultFetchBytesOptions + fetchOpts.TargetPlatform = targetPlatform + desc, content, err = oras.FetchBytes(ctx, manifests, opts.targetRef, fetchOpts) if err != nil { return err } diff --git a/cmd/oras/pull.go b/cmd/oras/pull.go index 0af49aa33..3b4ffb7ec 100644 --- a/cmd/oras/pull.go +++ b/cmd/oras/pull.go @@ -18,7 +18,6 @@ package main import ( "context" "fmt" - "os" "sync" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -26,20 +25,18 @@ import ( "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/content/oci" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" - "oras.land/oras/internal/cache" "oras.land/oras/internal/docker" ) type pullOptions struct { + option.Cache option.Common option.Remote targetRef string - cacheRoot string KeepOldFiles bool PathTraversal bool Output string @@ -68,7 +65,6 @@ Example - Pull files with local cache: `, Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { - opts.cacheRoot = os.Getenv("ORAS_CACHE") return opts.ReadPassword() }, RunE: func(cmd *cobra.Command, args []string) error { @@ -94,13 +90,9 @@ func runPull(opts pullOptions) error { if repo.Reference.Reference == "" { return errors.NewErrInvalidReference(repo.Reference) } - var src oras.ReadOnlyTarget = repo - if opts.cacheRoot != "" { - ociStore, err := oci.New(opts.cacheRoot) - if err != nil { - return err - } - src = cache.New(repo, ociStore) + src, err := opts.CachedTarget(repo) + if err != nil { + return err } // Copy Options From b68cfaec97415a0dda537969dac4eb3744269d13 Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Fri, 16 Sep 2022 17:17:43 +0800 Subject: [PATCH 09/11] temporary change Signed-off-by: Haoliang Yue --- cmd/oras/internal/option/cache.go | 8 ++++---- cmd/oras/internal/option/cache_test.go | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/oras/internal/option/cache.go b/cmd/oras/internal/option/cache.go index d80f70bc2..0012a4253 100644 --- a/cmd/oras/internal/option/cache.go +++ b/cmd/oras/internal/option/cache.go @@ -24,14 +24,14 @@ import ( ) type Cache struct { - CacheRoot string + Root string } // CachedTarget gets the target storage with caching if cache root is specified. func (opts *Cache) CachedTarget(src oras.ReadOnlyTarget) (oras.ReadOnlyTarget, error) { - opts.CacheRoot = os.Getenv("ORAS_CACHE") - if opts.CacheRoot != "" { - ociStore, err := oci.New(opts.CacheRoot) + opts.Root = os.Getenv("ORAS_CACHE") + if opts.Root != "" { + ociStore, err := oci.New(opts.Root) if err != nil { return nil, err } diff --git a/cmd/oras/internal/option/cache_test.go b/cmd/oras/internal/option/cache_test.go index a910cc8eb..6b8b6663a 100644 --- a/cmd/oras/internal/option/cache_test.go +++ b/cmd/oras/internal/option/cache_test.go @@ -31,6 +31,7 @@ var mockTarget oras.ReadOnlyTarget = memory.New() func TestCache_CachedTarget(t *testing.T) { tempDir := t.TempDir() os.Setenv("ORAS_CACHE", tempDir) + defer os.Unsetenv("ORAS_CACHE") opts := Cache{} ociStore, err := oci.New(tempDir) From 0952ea9d67bb266e700224778dcf980288c2dce2 Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Fri, 16 Sep 2022 18:20:19 +0800 Subject: [PATCH 10/11] resolve comments Signed-off-by: Haoliang Yue --- cmd/oras/manifest/fetch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index 6a21cdd89..bd616739d 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -109,7 +109,6 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { } var desc ocispec.Descriptor - var content []byte if opts.OutputDescriptor && opts.outputPath == "" { // fetch manifest descriptor only desc, err = oras.Resolve(ctx, manifests, opts.targetRef, oras.DefaultResolveOptions) @@ -118,6 +117,7 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { } } else { // fetch manifest content + var content []byte fetchOpts := oras.DefaultFetchBytesOptions fetchOpts.TargetPlatform = targetPlatform desc, content, err = oras.FetchBytes(ctx, manifests, opts.targetRef, fetchOpts) @@ -125,8 +125,8 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { return err } - // outputs manifest content if opts.outputPath == "" || opts.outputPath == "-" { + // output manifest content return opts.Output(os.Stdout, content) } From e3d4aff49107c5725f89b9bece080e9772cd81e5 Mon Sep 17 00:00:00 2001 From: Haoliang Yue Date: Fri, 16 Sep 2022 18:24:20 +0800 Subject: [PATCH 11/11] make a nit change Signed-off-by: Haoliang Yue --- cmd/oras/manifest/fetch.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/oras/manifest/fetch.go b/cmd/oras/manifest/fetch.go index bd616739d..8e52cca96 100644 --- a/cmd/oras/manifest/fetch.go +++ b/cmd/oras/manifest/fetch.go @@ -142,10 +142,7 @@ func fetchManifest(opts fetchOptions) (fetchErr error) { if err != nil { return err } - err = opts.Output(os.Stdout, descBytes) - if err != nil { - return err - } + return opts.Output(os.Stdout, descBytes) } return nil