From eb25c8d77efe5784e64f56909c0d4dc2860c16dc Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 03:32:12 +0000 Subject: [PATCH 01/44] feat: pack OCI image spec v1.1 manifests Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 26 +++++++---------- cmd/oras/root/attach.go | 3 +- cmd/oras/root/push.go | 22 +++++++------- go.mod | 4 +-- go.sum | 8 ++--- internal/graph/graph.go | 28 +----------------- internal/graph/graph_test.go | 50 ++++---------------------------- 7 files changed, 35 insertions(+), 106 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index ccbb204fb..f403f9813 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -18,35 +18,31 @@ package option import ( "fmt" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/pflag" ) +const ( + V1_1 = "1.1" + V1_0 = "1.0" +) + // ImageSpec option struct. type ImageSpec struct { - // Manifest type for building artifact - ManifestMediaType string - - // specFlag should be provided in form of `-` - specFlag string + Flag string } // Parse parses flags into the option. func (opts *ImageSpec) Parse() error { - switch opts.specFlag { - case "v1.1-image": - opts.ManifestMediaType = ocispec.MediaTypeImageManifest - case "v1.1-artifact": - opts.ManifestMediaType = ocispec.MediaTypeArtifactManifest - default: - return fmt.Errorf("unknown image specification flag: %q", opts.specFlag) + switch opts.Flag { + case V1_1, V1_0: + return nil } - return nil + return fmt.Errorf("unknown image specification flag: %q", opts.Flag) } // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.specFlag, "image-spec", "v1.1-image", "[Experimental] specify manifest type for building artifact. options: v1.1-image, v1.1-artifact") + fs.StringVar(&opts.Flag, "image-spec", V1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", V1_1, V1_0)) } // distributionSpec option struct. diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index d0a84dd13..a18be3e0d 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -35,7 +35,6 @@ import ( type attachOptions struct { option.Common option.Packer - option.ImageSpec option.Target option.Referrers @@ -139,7 +138,7 @@ func runAttach(ctx context.Context, opts attachOptions) error { packOpts := oras.PackOptions{ Subject: &subject, ManifestAnnotations: annotations[option.AnnotationManifest], - PackImageManifest: opts.ManifestMediaType == ocispec.MediaTypeImageManifest, + PackImageManifest: true, } pack := func() (ocispec.Descriptor, error) { return oras.Pack(ctx, store, opts.artifactType, descs, packOpts) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 0423df717..b0d95bfd4 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -102,15 +102,16 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t opts.RawReference = refs[0] opts.extraRefs = refs[1:] opts.FileRefs = args[1:] - if opts.manifestConfigRef != "" { - if opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided") - } - if opts.ManifestMediaType == ocispec.MediaTypeArtifactManifest { - return errors.New("cannot build an OCI artifact with manifest config") - } + + if err := option.Parse(&opts); err != nil { + return err + } + + if opts.ImageSpec.Flag == option.V1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") } - return option.Parse(&opts) + return nil + }, RunE: func(cmd *cobra.Command, args []string) error { return runPush(cmd.Context(), opts) @@ -135,6 +136,7 @@ func runPush(ctx context.Context, opts pushOptions) error { packOpts := oras.PackOptions{ ConfigAnnotations: annotations[option.AnnotationConfig], ManifestAnnotations: annotations[option.AnnotationManifest], + PackImageManifest: true, } store, err := file.New("") if err != nil { @@ -152,10 +154,6 @@ func runPush(ctx context.Context, opts pushOptions) error { } desc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &desc - packOpts.PackImageManifest = true - } - if opts.ManifestMediaType == ocispec.MediaTypeImageManifest { - packOpts.PackImageManifest = true } descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { diff --git a/go.mod b/go.mod index d76346c6e..1f32276f6 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.20 require ( github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0-rc4 github.com/oras-project/oras-credentials-go v0.2.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 golang.org/x/term v0.10.0 gopkg.in/yaml.v3 v3.0.1 - oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b + oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 ) require ( diff --git a/go.sum b/go.sum index ef1f32966..66806f226 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/oras-project/oras-credentials-go v0.2.0 h1:BvWAXo0e5unWR6Hfxyb0K04mHNHreQz/Zclw6IzCYJo= github.com/oras-project/oras-credentials-go v0.2.0/go.mod h1:JVdg7a5k7hzTrEeeouwag0aCv7OLrS77r7/6w3gVirU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -34,5 +34,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b h1:NYynybpqtG3lLTZMWNlrvUlcyGakCke57tg4TX6w2kA= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b/go.mod h1:goptA58HogB/6sLN7KV6FgoiurcVxd5QBqv8wMVB0as= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 h1:lK/VtXN9+dzSD3EMuCXu0D28OJadBxB9yUpaJBDIABM= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ= diff --git a/internal/graph/graph.go b/internal/graph/graph.go index bcefdd43d..731df5faf 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -43,18 +43,6 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr nodes = manifest.Layers subject = manifest.Subject config = &manifest.Config - case ocispec.MediaTypeArtifactManifest: - var fetched []byte - fetched, err = content.FetchAll(ctx, fetcher, node) - if err != nil { - return - } - var manifest ocispec.Artifact - if err = json.Unmarshal(fetched, &manifest); err != nil { - return - } - nodes = manifest.Blobs - subject = manifest.Subject default: nodes, err = content.Successors(ctx, fetcher, node) } @@ -83,20 +71,6 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc } for _, node := range predecessors { switch node.MediaType { - case ocispec.MediaTypeArtifactManifest: - fetched, err := fetchBytes(ctx, target, node) - if err != nil { - return nil, err - } - var artifact ocispec.Artifact - if err := json.Unmarshal(fetched, &artifact); err != nil { - return nil, err - } - if artifact.Subject == nil || !content.Equal(*artifact.Subject, desc) { - continue - } - node.ArtifactType = artifact.ArtifactType - node.Annotations = artifact.Annotations case ocispec.MediaTypeImageManifest: fetched, err := fetchBytes(ctx, target, node) if err != nil { @@ -141,7 +115,7 @@ func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStor } for _, node := range predecessors { switch node.MediaType { - case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: + case ocispec.MediaTypeImageManifest: results = append(results, node) } } diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 599307fa2..1c7e14ab3 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -89,19 +89,6 @@ func TestReferrers(t *testing.T) { } appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) } - generateArtifact := func(artifactType string, subject *ocispec.Descriptor, annotations map[string]string, blobs ...ocispec.Descriptor) { - manifest := ocispec.Artifact{ - Subject: subject, - Blobs: blobs, - Annotations: annotations, - ArtifactType: artifactType, - } - manifestJSON, err := json.Marshal(manifest) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON) - } generateIndex := func(manifests ...ocispec.Descriptor) { index := ocispec.Index{ Manifests: manifests, @@ -117,11 +104,10 @@ func TestReferrers(t *testing.T) { imgConfig subject image - artifact index ) anno := map[string]string{"test": "foo"} - appendBlob(ocispec.MediaTypeArtifactManifest, []byte(`{"name":"subject content"}`)) + appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) imageType := "test.image" appendBlob(imageType, []byte("config content")) generateImage(nil, nil, descs[imgConfig], descs[blob]) @@ -129,12 +115,7 @@ func TestReferrers(t *testing.T) { imageDesc := descs[image] imageDesc.Annotations = anno imageDesc.ArtifactType = imageType - artifactType := "test.artifact" - generateArtifact(artifactType, &descs[subject], anno, descs[blob]) generateIndex(descs[subject]) - artifactDesc := descs[artifact] - artifactDesc.Annotations = anno - artifactDesc.ArtifactType = artifactType referrers := []ocispec.Descriptor{descs[image], descs[image]} memory := memory.New() @@ -163,7 +144,6 @@ func TestReferrers(t *testing.T) { {"should return nil for config node", args{ctx, finder, descs[imgConfig], ""}, nil, false}, {"should return nil for blob/layer node", args{ctx, finder, descs[blob], ""}, nil, false}, {"should find filtered image referrer", args{ctx, finder, descs[subject], imageType}, []ocispec.Descriptor{imageDesc}, false}, - {"should find filtered artifact referrer", args{ctx, finder, descs[subject], artifactType}, []ocispec.Descriptor{artifactDesc}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -179,15 +159,14 @@ func TestReferrers(t *testing.T) { } t.Run("should find referrers in predecessors", func(t *testing.T) { - want1 := []ocispec.Descriptor{artifactDesc, imageDesc} - want2 := []ocispec.Descriptor{imageDesc, artifactDesc} + want := []ocispec.Descriptor{imageDesc} got, err := Referrers(ctx, finder, descs[subject], "") if err != nil { t.Errorf("Referrers() error = %v", err) return } - if !reflect.DeepEqual(got, want1) && !reflect.DeepEqual(got, want2) { - t.Errorf("Referrers() = %v, want %v", got, want1) + if !reflect.DeepEqual(got, want) && !reflect.DeepEqual(got, want) { + t.Errorf("Referrers() = %v, want %v", got, want) } }) } @@ -216,18 +195,6 @@ func TestSuccessors(t *testing.T) { } appendBlob(mediaType, manifestJSON) } - generateArtifact := func(artifactType string, subject *ocispec.Descriptor, blobs ...ocispec.Descriptor) { - manifest := ocispec.Artifact{ - MediaType: ocispec.MediaTypeArtifactManifest, - Subject: subject, - Blobs: blobs, - } - manifestJSON, err := json.Marshal(manifest) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON) - } generateIndex := func(manifests ...ocispec.Descriptor) { index := ocispec.Index{ Manifests: manifests, @@ -243,16 +210,13 @@ func TestSuccessors(t *testing.T) { config ociImage dockerImage - artifact index ) - appendBlob(ocispec.MediaTypeArtifactManifest, []byte(`{"name":"subject content"}`)) + appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) imageType := "test.image" appendBlob(imageType, []byte("config content")) generateImage(&descs[subject], ocispec.MediaTypeImageManifest, descs[config]) generateImage(&descs[subject], docker.MediaTypeManifest, descs[config]) - artifactType := "test.artifact" - generateArtifact(artifactType, &descs[subject]) generateIndex(descs[subject]) memory := memory.New() ctx := context.Background() @@ -276,12 +240,10 @@ func TestSuccessors(t *testing.T) { wantConfig *ocispec.Descriptor wantErr bool }{ - {"should failed to get non-existent artifact", args{ctx, fetcher, ocispec.Descriptor{MediaType: ocispec.MediaTypeArtifactManifest}}, nil, nil, nil, true}, {"should failed to get non-existent OCI image", args{ctx, fetcher, ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest}}, nil, nil, nil, true}, {"should failed to get non-existent docker image", args{ctx, fetcher, ocispec.Descriptor{MediaType: docker.MediaTypeManifest}}, nil, nil, nil, true}, {"should get success of a docker image", args{ctx, fetcher, descs[dockerImage]}, nil, &descs[subject], &descs[config], false}, {"should get success of an OCI image", args{ctx, fetcher, descs[ociImage]}, nil, &descs[subject], &descs[config], false}, - {"should get success of an artifact", args{ctx, fetcher, descs[artifact]}, nil, &descs[subject], nil, false}, {"should get success of an index", args{ctx, fetcher, descs[index]}, []ocispec.Descriptor{descs[subject]}, nil, nil, false}, } for _, tt := range tests { @@ -345,7 +307,7 @@ func TestFindReferrerPredecessors(t *testing.T) { image ) var anno map[string]string - appendBlob(ocispec.MediaTypeArtifactManifest, []byte(`{"name":"subject content"}`)) + appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) imageType := "test.image" appendBlob(imageType, []byte("config content")) generateImage(&descs[subject], anno, descs[imgConfig]) From f8be004f5bf8c2dde9ab4dd88bc5fe27cb461730 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 03:38:49 +0000 Subject: [PATCH 02/44] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/attach.go | 44 +++----------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 883ab2fed..6e714c024 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -111,9 +111,7 @@ var _ = Describe("Common registry users:", func() { subjectRef := RegistryRef(Host, testRepo, foobar.Tag) prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). - WithWorkDir(tempDir). - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() + MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() // validate var index ocispec.Index @@ -148,24 +146,6 @@ var _ = Describe("Common registry users:", func() { ExpectFailure(). Exec() }) - - It("should attach a file via a OCI Artifact", func() { - testRepo := attachTestRepo("artifact") - tempDir := PrepareTempFiles() - subjectRef := RegistryRef(Host, testRepo, foobar.Tag) - prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef) - // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-artifact"). - WithWorkDir(tempDir). - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() - - // validate - var index ocispec.Index - bytes := ORAS("discover", subjectRef, "-o", "json").Exec().Out.Contents() - Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) - Expect(len(index.Manifests)).To(Equal(1)) - Expect(index.Manifests[0].MediaType).To(Equal("application/vnd.oci.artifact.manifest.v1+json")) - }) }) }) @@ -177,7 +157,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() @@ -260,7 +240,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image", "--distribution-spec", "v1.1-referrers-tag"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--distribution-spec", "v1.1-referrers-tag"). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() @@ -323,7 +303,7 @@ var _ = Describe("OCI image layout users:", func() { subjectRef := LayoutRef(root, foobar.Tag) prepare(root) // test - ORAS("attach", "--artifact-type", "test.attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(root). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() @@ -334,21 +314,5 @@ var _ = Describe("OCI image layout users:", func() { Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].MediaType).To(Equal("application/vnd.oci.image.manifest.v1+json")) }) - It("should attach a file via a OCI Artifact", func() { - root := PrepareTempFiles() - subjectRef := LayoutRef(root, foobar.Tag) - prepare(root) - // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, Flags.Layout, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-artifact"). - WithWorkDir(root). - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() - - // validate - var index ocispec.Index - bytes := ORAS("discover", subjectRef, Flags.Layout, "-o", "json").Exec().Out.Contents() - Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) - Expect(len(index.Manifests)).To(Equal(1)) - Expect(index.Manifests[0].MediaType).To(Equal("application/vnd.oci.artifact.manifest.v1+json")) - }) }) }) From 125f1852fc10b56c3180f053c82f8a3770845dd8 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 03:44:45 +0000 Subject: [PATCH 03/44] fix line Signed-off-by: Billy Zha --- internal/graph/graph_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 1c7e14ab3..370d34bb2 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -165,7 +165,7 @@ func TestReferrers(t *testing.T) { t.Errorf("Referrers() error = %v", err) return } - if !reflect.DeepEqual(got, want) && !reflect.DeepEqual(got, want) { + if !reflect.DeepEqual(got, want) { t.Errorf("Referrers() = %v, want %v", got, want) } }) From e65ee375863c526067572bf338f2fd3c86de6d6f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 04:36:35 +0000 Subject: [PATCH 04/44] fix e2e Signed-off-by: Billy Zha --- internal/graph/graph.go | 69 +++++++++++++++++++++++++++++- test/e2e/suite/command/attach.go | 11 ++--- test/e2e/suite/command/discover.go | 2 +- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 731df5faf..77c054a59 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -25,6 +25,34 @@ import ( "oras.land/oras/internal/docker" ) +// MediaTypeArtifactManifest specifies the media type for a content descriptor. +const MediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json" + +// Artifact describes an artifact manifest. +// This structure provides `application/vnd.oci.artifact.manifest.v1+json` mediatype when marshalled to JSON. +// +// This manifest type was introduced in image-spec v1.1.0-rc1 and was removed in +// image-spec v1.1.0-rc3. It is not part of the current image-spec and is kept +// here for Go compatibility. +// +// Reference: https://github.com/opencontainers/image-spec/pull/999 +type Artifact struct { + // MediaType is the media type of the object this schema refers to. + MediaType string `json:"mediaType"` + + // ArtifactType is the IANA media type of the artifact this schema refers to. + ArtifactType string `json:"artifactType"` + + // Blobs is a collection of blobs referenced by this manifest. + Blobs []ocispec.Descriptor `json:"blobs,omitempty"` + + // Subject (reference) is an optional link from the artifact to another manifest forming an association between the artifact and the other manifest. + Subject *ocispec.Descriptor `json:"subject,omitempty"` + + // Annotations contains arbitrary metadata for the artifact manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + // Successors returns the nodes directly pointed by the current node, picking // out subject and config descriptor if applicable. // Returning nil when no subject and config found. @@ -43,6 +71,18 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr nodes = manifest.Layers subject = manifest.Subject config = &manifest.Config + case MediaTypeArtifactManifest: + var fetched []byte + fetched, err = content.FetchAll(ctx, fetcher, node) + if err != nil { + return + } + var manifest Artifact + if err = json.Unmarshal(fetched, &manifest); err != nil { + return + } + nodes = manifest.Blobs + subject = manifest.Subject default: nodes, err = content.Successors(ctx, fetcher, node) } @@ -71,6 +111,20 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc } for _, node := range predecessors { switch node.MediaType { + case MediaTypeArtifactManifest: + fetched, err := fetchBytes(ctx, target, node) + if err != nil { + return nil, err + } + var artifact Artifact + if err := json.Unmarshal(fetched, &artifact); err != nil { + return nil, err + } + if artifact.Subject == nil || !content.Equal(*artifact.Subject, desc) { + continue + } + node.ArtifactType = artifact.ArtifactType + node.Annotations = artifact.Annotations case ocispec.MediaTypeImageManifest: fetched, err := fetchBytes(ctx, target, node) if err != nil { @@ -115,8 +169,21 @@ func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStor } for _, node := range predecessors { switch node.MediaType { - case ocispec.MediaTypeImageManifest: + case MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: results = append(results, node) + case ocispec.MediaTypeImageIndex: + fetched, err := fetchBytes(ctx, src, node) + if err != nil { + return nil, err + } + // convert to json + var index ocispec.Index + if err := json.Unmarshal(fetched, &index); err != nil { + return nil, err + } + if index.Subject != nil && content.Equal(*index.Subject, desc) { + results = append(results, node) + } } } return results, nil diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 6e714c024..18f2208a6 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -43,7 +43,6 @@ var _ = Describe("ORAS beginners:", func() { It("should show preview and help doc", func() { out := ORAS("attach", "--help").MatchKeyWords(feature.Preview.Mark+" Attach", feature.Preview.Description, ExampleDesc).Exec() gomega.Expect(out).Should(gbytes.Say("--distribution-spec string\\s+%s", regexp.QuoteMeta(feature.Preview.Mark))) - gomega.Expect(out).Should(gbytes.Say("--image-spec string\\s+%s", regexp.QuoteMeta(feature.Experimental.Mark))) }) It("should fail when no subject reference provided", func() { @@ -65,11 +64,6 @@ var _ = Describe("ORAS beginners:", func() { ORAS("attach", "--artifact-type", "oras.test", RegistryRef(Host, ImageRepo, foobar.Tag), "--distribution-spec", "???"). ExpectFailure().MatchErrKeyWords("unknown distribution specification flag").Exec() }) - - It("should fail if image spec is unkown", func() { - ORAS("attach", "--artifact-type", "oras.test", RegistryRef(Host, ImageRepo, foobar.Tag), "--image-spec", "???"). - ExpectFailure().MatchErrKeyWords("unknown image specification flag").Exec() - }) }) }) @@ -111,8 +105,9 @@ var _ = Describe("Common registry users:", func() { subjectRef := RegistryRef(Host, testRepo, foobar.Tag) prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef) // test - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() - + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). + WithWorkDir(tempDir). + MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() // validate var index ocispec.Index bytes := ORAS("discover", subjectRef, "-o", "json").Exec().Out.Contents() diff --git a/test/e2e/suite/command/discover.go b/test/e2e/suite/command/discover.go index c753edfce..22d717d02 100644 --- a/test/e2e/suite/command/discover.go +++ b/test/e2e/suite/command/discover.go @@ -262,7 +262,7 @@ var _ = Describe("OCI image layout users:", func() { When("running discover command with table output", func() { format := "table" - It("should direct referrers of a subject", func() { + It("should get direct referrers of a subject", func() { referrers := []ocispec.Descriptor{foobar.SBOMImageReferrer, foobar.SBOMArtifactReferrer} // prepare subjectRef := LayoutRef(GinkgoT().TempDir(), foobar.Tag) From e068e8da7aa889337e6cc7b6ce8e7b02663bc539 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 04:46:42 +0000 Subject: [PATCH 05/44] use pack type Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 20 +++++++++++++------- cmd/oras/root/push.go | 3 ++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index f403f9813..fe1059e3e 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -19,30 +19,36 @@ import ( "fmt" "github.com/spf13/pflag" + "oras.land/oras-go/v2" ) const ( - V1_1 = "1.1" + v1_1 = "1.1" V1_0 = "1.0" ) // ImageSpec option struct. type ImageSpec struct { - Flag string + flag string + PackType oras.PackManifestType } // Parse parses flags into the option. func (opts *ImageSpec) Parse() error { - switch opts.Flag { - case V1_1, V1_0: - return nil + switch opts.flag { + case v1_1: + opts.PackType = oras.PackManifestTypeImageV1_1_0_RC4 + case V1_0: + opts.PackType = oras.PackManifestTypeImageV1_1_0_RC2 + default: + return fmt.Errorf("unknown image specification flag: %q", opts.flag) } - return fmt.Errorf("unknown image specification flag: %q", opts.Flag) + return nil } // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.Flag, "image-spec", V1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", V1_1, V1_0)) + fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, V1_0)) } // distributionSpec option struct. diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index b0d95bfd4..bd6467b05 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -107,7 +107,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return err } - if opts.ImageSpec.Flag == option.V1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { + if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC2 && opts.manifestConfigRef != "" && opts.artifactType != "" { return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") } return nil @@ -137,6 +137,7 @@ func runPush(ctx context.Context, opts pushOptions) error { ConfigAnnotations: annotations[option.AnnotationConfig], ManifestAnnotations: annotations[option.AnnotationManifest], PackImageManifest: true, + PackManifestType: opts.ImageSpec.PackType, } store, err := file.New("") if err != nil { From 9520375ba4228f8929616f2f11c9c0ad74848eba Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 12:52:51 +0000 Subject: [PATCH 06/44] update e2e go mod Signed-off-by: Billy Zha --- test/e2e/go.mod | 4 ++-- test/e2e/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 09b86ff2c..9f1bee561 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -6,9 +6,9 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0-rc4 gopkg.in/yaml.v2 v2.4.0 - oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b + oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 ) require ( diff --git a/test/e2e/go.sum b/test/e2e/go.sum index bac81e704..350be0911 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -20,8 +20,8 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -47,5 +47,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b h1:NYynybpqtG3lLTZMWNlrvUlcyGakCke57tg4TX6w2kA= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b/go.mod h1:goptA58HogB/6sLN7KV6FgoiurcVxd5QBqv8wMVB0as= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 h1:lK/VtXN9+dzSD3EMuCXu0D28OJadBxB9yUpaJBDIABM= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ= From 84e3b1678590c3383d9b6925b399908b881f8514 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 00:46:24 +0000 Subject: [PATCH 07/44] feat: support warning in remote targets Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 24 ++++++++++++++++++++++-- cmd/oras/internal/option/remote_test.go | 6 +++--- cmd/oras/internal/option/target.go | 6 +++--- cmd/oras/root/attach.go | 2 +- cmd/oras/root/blob/delete.go | 2 +- cmd/oras/root/blob/push.go | 4 ++-- cmd/oras/root/cp.go | 2 +- cmd/oras/root/login.go | 2 +- cmd/oras/root/manifest/delete.go | 4 ++-- cmd/oras/root/manifest/push.go | 2 +- cmd/oras/root/push.go | 4 ++-- cmd/oras/root/repo/ls.go | 4 ++-- cmd/oras/root/tag.go | 4 ++-- 13 files changed, 43 insertions(+), 23 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index efd4e054a..64f2f769a 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -25,6 +25,7 @@ import ( "os" "strconv" "strings" + "sync" credentials "github.com/oras-project/oras-credentials-go" "github.com/spf13/pflag" @@ -248,13 +249,22 @@ func (opts *Remote) Credential() auth.Credential { } // NewRegistry assembles a oras remote registry. -func (opts *Remote) NewRegistry(hostname string, common Common) (reg *remote.Registry, err error) { +func (opts *Remote) NewRegistry(hostname string, log func(...interface{}), common Common) (reg *remote.Registry, err error) { reg, err = remote.NewRegistry(hostname) if err != nil { return nil, err } hostname = reg.Reference.Registry reg.PlainHTTP = opts.isPlainHttp(hostname) + + if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI != false { + once := sync.Once{} + reg.HandleWarning = func(warning remote.Warning) { + once.Do(func() { + log(warning.Text) + }) + } + } if reg.Client, err = opts.authClient(hostname, common.Debug); err != nil { return nil, err } @@ -262,7 +272,7 @@ func (opts *Remote) NewRegistry(hostname string, common Common) (reg *remote.Reg } // NewRepository assembles a oras remote repository. -func (opts *Remote) NewRepository(reference string, common Common) (repo *remote.Repository, err error) { +func (opts *Remote) NewRepository(reference string, log func(...interface{}), common Common) (repo *remote.Repository, err error) { repo, err = remote.NewRepository(reference) if err != nil { return nil, err @@ -272,6 +282,16 @@ func (opts *Remote) NewRepository(reference string, common Common) (repo *remote if repo.Client, err = opts.authClient(hostname, common.Debug); err != nil { return nil, err } + + if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI != false { + once := sync.Once{} + repo.HandleWarning = func(warning remote.Warning) { + once.Do(func() { + log(warning.Text) + }) + } + } + if opts.distributionSpec.referrersAPI != nil { if err := repo.SetReferrersCapability(*opts.distributionSpec.referrersAPI); err != nil { return nil, err diff --git a/cmd/oras/internal/option/remote_test.go b/cmd/oras/internal/option/remote_test.go index abd1c3029..f7b105091 100644 --- a/cmd/oras/internal/option/remote_test.go +++ b/cmd/oras/internal/option/remote_test.go @@ -180,7 +180,7 @@ func TestRemote_NewRegistry(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - reg, err := opts.NewRegistry(uri.Host, opts.Common) + reg, err := opts.NewRegistry(uri.Host, nil, opts.Common) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -208,7 +208,7 @@ func TestRemote_NewRepository(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - repo, err := opts.NewRepository(uri.Host+"/"+testRepo, opts.Common) + repo, err := opts.NewRepository(uri.Host+"/"+testRepo, nil, opts.Common) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -255,7 +255,7 @@ func TestRemote_NewRepository_Retry(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - repo, err := opts.NewRepository(uri.Host+"/"+testRepo, opts.Common) + repo, err := opts.NewRepository(uri.Host+"/"+testRepo, nil, opts.Common) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index 9494e1789..c94baf37c 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -111,7 +111,7 @@ func parseOCILayoutReference(raw string) (path string, ref string, err error) { } // NewTarget generates a new target based on opts. -func (opts *Target) NewTarget(common Common) (oras.GraphTarget, error) { +func (opts *Target) NewTarget(common Common, log func(...interface{})) (oras.GraphTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error @@ -121,7 +121,7 @@ func (opts *Target) NewTarget(common Common) (oras.GraphTarget, error) { } return oci.New(opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, common) + repo, err := opts.NewRepository(opts.RawReference, nil, common) if err != nil { return nil, err } @@ -159,7 +159,7 @@ func (opts *Target) NewReadonlyTarget(ctx context.Context, common Common) (ReadO } return oci.NewFromTar(ctx, opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, common) + repo, err := opts.NewRepository(opts.RawReference, nil, common) if err != nil { return nil, err } diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index a18be3e0d..f962c840a 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -116,7 +116,7 @@ func runAttach(ctx context.Context, opts attachOptions) error { } defer store.Close() - dst, err := opts.NewTarget(opts.Common) + dst, err := opts.NewTarget(opts.Common, logger.Warn) if err != nil { return err } diff --git a/cmd/oras/root/blob/delete.go b/cmd/oras/root/blob/delete.go index 0f55d044b..7e9d46bf8 100644 --- a/cmd/oras/root/blob/delete.go +++ b/cmd/oras/root/blob/delete.go @@ -72,7 +72,7 @@ Example - Delete a blob and print its descriptor: func deleteBlob(ctx context.Context, opts deleteBlobOptions) (err error) { ctx, _ = opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, opts.Common) + repo, err := opts.NewRepository(opts.targetRef, nil, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/blob/push.go b/cmd/oras/root/blob/push.go index 9de6c45fe..f973042a4 100644 --- a/cmd/oras/root/blob/push.go +++ b/cmd/oras/root/blob/push.go @@ -96,9 +96,9 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir': } func pushBlob(ctx context.Context, opts pushBlobOptions) (err error) { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewTarget(opts.Common) + repo, err := opts.NewTarget(opts.Common, logger.Warn) if err != nil { return err } diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 20cc2a753..e207391b0 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -111,7 +111,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { } // Prepare destination - dst, err := opts.To.NewTarget(opts.Common) + dst, err := opts.To.NewTarget(opts.Common, logger.Warn) if err != nil { return err } diff --git a/cmd/oras/root/login.go b/cmd/oras/root/login.go index aa862fd4d..7bed32686 100644 --- a/cmd/oras/root/login.go +++ b/cmd/oras/root/login.go @@ -108,7 +108,7 @@ func runLogin(ctx context.Context, opts loginOptions) (err error) { if err != nil { return err } - remote, err := opts.Remote.NewRegistry(opts.Hostname, opts.Common) + remote, err := opts.Remote.NewRegistry(opts.Hostname, nil, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/delete.go b/cmd/oras/root/manifest/delete.go index 8ece10400..1aae58a64 100644 --- a/cmd/oras/root/manifest/delete.go +++ b/cmd/oras/root/manifest/delete.go @@ -76,8 +76,8 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 } func deleteManifest(ctx context.Context, opts deleteOptions) error { - ctx, _ = opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, opts.Common) + ctx, logger := opts.WithContext(ctx) + repo, err := opts.NewRepository(opts.targetRef, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/push.go b/cmd/oras/root/manifest/push.go index f6391c939..223cf4274 100644 --- a/cmd/oras/root/manifest/push.go +++ b/cmd/oras/root/manifest/push.go @@ -109,7 +109,7 @@ func pushManifest(ctx context.Context, opts pushOptions) error { ctx, logger := opts.WithContext(ctx) var target oras.Target var err error - target, err = opts.NewTarget(opts.Common) + target, err = opts.NewTarget(opts.Common, logger.Warn) if err != nil { return err } diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index bd6467b05..e40d4cb31 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -126,7 +126,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t } func runPush(ctx context.Context, opts pushOptions) error { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) annotations, err := opts.LoadManifestAnnotations() if err != nil { return err @@ -173,7 +173,7 @@ func runPush(ctx context.Context, opts pushOptions) error { } // prepare push - dst, err := opts.NewTarget(opts.Common) + dst, err := opts.NewTarget(opts.Common, logger.Warn) if err != nil { return err } diff --git a/cmd/oras/root/repo/ls.go b/cmd/oras/root/repo/ls.go index f2da5e5f3..e72298401 100644 --- a/cmd/oras/root/repo/ls.go +++ b/cmd/oras/root/repo/ls.go @@ -69,8 +69,8 @@ Example - List the repositories under the registry that include values lexically } func listRepository(ctx context.Context, opts repositoryOptions) error { - ctx, _ = opts.WithContext(ctx) - reg, err := opts.Remote.NewRegistry(opts.hostname, opts.Common) + ctx, logger := opts.WithContext(ctx) + reg, err := opts.Remote.NewRegistry(opts.hostname, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/tag.go b/cmd/oras/root/tag.go index 35475fec6..ecb486a8b 100644 --- a/cmd/oras/root/tag.go +++ b/cmd/oras/root/tag.go @@ -72,8 +72,8 @@ Example - Tag the manifest 'v1.0.1' to 'v1.0.2' in an OCI image layout folder 'l } func tagManifest(ctx context.Context, opts tagOptions) error { - ctx, _ = opts.WithContext(ctx) - target, err := opts.NewTarget(opts.Common) + ctx, logger := opts.WithContext(ctx) + target, err := opts.NewTarget(opts.Common, logger.Warn) if err != nil { return err } From 09755f2df6ea527ed77d450c6ce4b16ac4182101 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 01:09:06 +0000 Subject: [PATCH 08/44] remove PackManifestTypeImageV1_1_0_RC2 Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 11 ++++++----- cmd/oras/root/push.go | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index fe1059e3e..625a1e175 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -23,8 +23,9 @@ import ( ) const ( - v1_1 = "1.1" - V1_0 = "1.0" + v1_1 = "1.1" + v1_0 = "1.0" + PackManifestTypeImageV1_0 = 0 ) // ImageSpec option struct. @@ -38,8 +39,8 @@ func (opts *ImageSpec) Parse() error { switch opts.flag { case v1_1: opts.PackType = oras.PackManifestTypeImageV1_1_0_RC4 - case V1_0: - opts.PackType = oras.PackManifestTypeImageV1_1_0_RC2 + case v1_0: + opts.PackType = PackManifestTypeImageV1_0 default: return fmt.Errorf("unknown image specification flag: %q", opts.flag) } @@ -48,7 +49,7 @@ func (opts *ImageSpec) Parse() error { // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, V1_0)) + fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, v1_0)) } // distributionSpec option struct. diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index bd6467b05..a1e93936b 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -107,7 +107,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return err } - if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC2 && opts.manifestConfigRef != "" && opts.artifactType != "" { + if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") } return nil From e298234bd4beb5409b1f6add7a8f75f4b04ab62f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 03:22:06 +0000 Subject: [PATCH 09/44] fix artifact push without subject Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index a1e93936b..52bbe1309 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -155,6 +155,12 @@ func runPush(ctx context.Context, opts pushOptions) error { } desc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &desc + } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { + configDesc := ocispec.DescriptorEmptyJSON + configDesc.MediaType = oras.MediaTypeUnknownConfig + //store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) + configDesc.Annotations = packOpts.ConfigAnnotations + packOpts.ConfigDescriptor = &configDesc } descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { From 3e0390603808af70c5308a7e442eba4784639fcd Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 03:27:13 +0000 Subject: [PATCH 10/44] add data Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 52bbe1309..fd59cb1e5 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -16,6 +16,7 @@ limitations under the License. package root import ( + "bytes" "context" "errors" "fmt" @@ -158,7 +159,7 @@ func runPush(ctx context.Context, opts pushOptions) error { } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { configDesc := ocispec.DescriptorEmptyJSON configDesc.MediaType = oras.MediaTypeUnknownConfig - //store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) + store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) configDesc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &configDesc } From 9e801c53370e987c273c10c3deed1f3435d67833 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 03:32:29 +0000 Subject: [PATCH 11/44] check returned error Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index fd59cb1e5..fc60c4ad3 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -159,7 +159,9 @@ func runPush(ctx context.Context, opts pushOptions) error { } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { configDesc := ocispec.DescriptorEmptyJSON configDesc.MediaType = oras.MediaTypeUnknownConfig - store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) + if err = store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)); err != nil { + return err + } configDesc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &configDesc } From 93b66b80956a6b5ef42db08af0874028fdce39bf Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 00:57:57 +0000 Subject: [PATCH 12/44] default to unknown artifact type when not provided Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 5 +++-- cmd/oras/root/push.go | 21 +++++++-------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index 625a1e175..988fa8e02 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -23,8 +23,9 @@ import ( ) const ( - v1_1 = "1.1" - v1_0 = "1.0" + v1_1 = "v1.1" + v1_0 = "v1.0" + // TODO: pending on https://github.com/oras-project/oras-go/issues/568 PackManifestTypeImageV1_0 = 0 ) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index fc60c4ad3..8239d06e0 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -16,7 +16,6 @@ limitations under the License. package root import ( - "bytes" "context" "errors" "fmt" @@ -103,16 +102,18 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t opts.RawReference = refs[0] opts.extraRefs = refs[1:] opts.FileRefs = args[1:] - if err := option.Parse(&opts); err != nil { return err } - - if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 { + if opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + } + if opts.manifestConfigRef == "" && opts.artifactType == "" { + opts.artifactType = oras.MediaTypeUnknownArtifact + } } return nil - }, RunE: func(cmd *cobra.Command, args []string) error { return runPush(cmd.Context(), opts) @@ -156,14 +157,6 @@ func runPush(ctx context.Context, opts pushOptions) error { } desc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &desc - } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { - configDesc := ocispec.DescriptorEmptyJSON - configDesc.MediaType = oras.MediaTypeUnknownConfig - if err = store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)); err != nil { - return err - } - configDesc.Annotations = packOpts.ConfigAnnotations - packOpts.ConfigDescriptor = &configDesc } descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { From 2117e45a0a1ed59a07f6907fd7a6d7c4a4c07af0 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:03:51 +0000 Subject: [PATCH 13/44] fix pack type Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 8239d06e0..edaa4b774 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -105,13 +105,11 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t if err := option.Parse(&opts); err != nil { return err } - if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 { - if opts.manifestConfigRef != "" && opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") - } - if opts.manifestConfigRef == "" && opts.artifactType == "" { - opts.artifactType = oras.MediaTypeUnknownArtifact - } + if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + } + if opts.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.manifestConfigRef == "" && opts.artifactType == "" { + opts.artifactType = oras.MediaTypeUnknownArtifact } return nil }, From 64b3ab588a92b4b893465f12e1d3849cff56cdca Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:12:37 +0000 Subject: [PATCH 14/44] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/push.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/suite/command/push.go b/test/e2e/suite/command/push.go index 17139fed4..12fd16ecc 100644 --- a/test/e2e/suite/command/push.go +++ b/test/e2e/suite/command/push.go @@ -47,7 +47,7 @@ var _ = Describe("Remote registry users:", func() { When("pushing to registy without OCI artifact support", func() { repoPrefix := fmt.Sprintf("command/push/%d", GinkgoRandomSeed()) statusKeys := []match.StateKey{ - foobar.ImageConfigStateKey("application/vnd.unknown.config.v1+json"), + foobar.ImageConfigStateKey("application/vnd.unknown.artifact.v1"), foobar.FileBarStateKey, } It("should push files without customized media types", func() { @@ -230,7 +230,7 @@ var _ = Describe("OCI image layout users:", func() { tag := "e2e" When("pushing to registy without OCI artifact support", func() { statusKeys := []match.StateKey{ - foobar.ImageConfigStateKey("application/vnd.unknown.config.v1+json"), + foobar.ImageConfigStateKey("application/vnd.oci.empty.v1+json"), foobar.FileBarStateKey, } From 552f65545b3827da81475a86f62efb26ca6eb75a Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:22:34 +0000 Subject: [PATCH 15/44] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite/command/push.go b/test/e2e/suite/command/push.go index 12fd16ecc..0fe788a79 100644 --- a/test/e2e/suite/command/push.go +++ b/test/e2e/suite/command/push.go @@ -47,7 +47,7 @@ var _ = Describe("Remote registry users:", func() { When("pushing to registy without OCI artifact support", func() { repoPrefix := fmt.Sprintf("command/push/%d", GinkgoRandomSeed()) statusKeys := []match.StateKey{ - foobar.ImageConfigStateKey("application/vnd.unknown.artifact.v1"), + foobar.ImageConfigStateKey("application/vnd.oci.empty.v1+json"), foobar.FileBarStateKey, } It("should push files without customized media types", func() { From 34da8f6ae2071418f70502f15e6254ffd75c032f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:37:03 +0000 Subject: [PATCH 16/44] remove examples Signed-off-by: Billy Zha --- cmd/oras/root/attach.go | 4 ---- cmd/oras/root/push.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index a18be3e0d..c539eb336 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -54,10 +54,6 @@ func attachCmd() *cobra.Command { Example - Attach file 'hi.txt' with type 'doc/example' to manifest 'hello:v1' in registry 'localhost:5000': oras attach --artifact-type doc/example localhost:5000/hello:v1 hi.txt -Example - Attach file "hi.txt" with specific media type when building the manifest: - oras attach --artifact-type doc/example --image-spec v1.1-image localhost:5000/hello:v1 hi.txt # OCI image - oras attach --artifact-type doc/example --image-spec v1.1-artifact localhost:5000/hello:v1 hi.txt # OCI artifact - Example - Attach file "hi.txt" using a specific method for the Referrers API: oras attach --artifact-type doc/example --distribution-spec v1.1-referrers-api localhost:5000/hello:v1 hi.txt # via API oras attach --artifact-type doc/example --distribution-spec v1.1-referrers-tag localhost:5000/hello:v1 hi.txt # via tag scheme diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index edaa4b774..0402f17ad 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -71,10 +71,6 @@ Example - Push file "hi.txt" with config type "application/vnd.me.config": Example - Push file "hi.txt" with the custom manifest config "config.json" of the custom media type "application/vnd.me.config": oras push --config config.json:application/vnd.me.config localhost:5000/hello:v1 hi.txt -Example - Push file "hi.txt" with specific media type when building the manifest: - oras push --image-spec v1.1-image localhost:5000/hello:v1 hi.txt # OCI image - oras push --image-spec v1.1-artifact localhost:5000/hello:v1 hi.txt # OCI artifact - Example - Push file to the insecure registry: oras push --insecure localhost:5000/hello:v1 hi.txt From 8c906c273896a8335d4e1b03d096bef21797696f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:42:07 +0000 Subject: [PATCH 17/44] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/attach.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 9071bccee..b0e5fba68 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -170,7 +170,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() From 88fd27259427289125bf8124a8a4730fa481fac2 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:42:59 +0000 Subject: [PATCH 18/44] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/attach.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 9071bccee..b0e5fba68 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -170,7 +170,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() From 098bcbcca124bdb581e2974094497c22e34b287d Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 02:30:54 +0000 Subject: [PATCH 19/44] fix merging Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 8 ++++---- cmd/oras/internal/option/target.go | 2 +- cmd/oras/root/attach.go | 2 +- cmd/oras/root/cp.go | 2 +- cmd/oras/root/manifest/push.go | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index dc2a747af..8483e13ce 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -249,7 +249,7 @@ func (opts *Remote) Credential() auth.Credential { } // NewRegistry assembles a oras remote registry. -func (opts *Remote) NewRegistry(hostname string, log func(...interface{}), common Common) (reg *remote.Registry, err error) { +func (opts *Remote) NewRegistry(hostname string, warn func(...interface{}), common Common) (reg *remote.Registry, err error) { reg, err = remote.NewRegistry(hostname) if err != nil { return nil, err @@ -261,7 +261,7 @@ func (opts *Remote) NewRegistry(hostname string, log func(...interface{}), commo once := sync.Once{} reg.HandleWarning = func(warning remote.Warning) { once.Do(func() { - log(warning.Text) + warn(warning.Text) }) } } @@ -272,7 +272,7 @@ func (opts *Remote) NewRegistry(hostname string, log func(...interface{}), commo } // NewRepository assembles a oras remote repository. -func (opts *Remote) NewRepository(reference string, log func(...interface{}), common Common) (repo *remote.Repository, err error) { +func (opts *Remote) NewRepository(reference string, warn func(...interface{}), common Common) (repo *remote.Repository, err error) { repo, err = remote.NewRepository(reference) if err != nil { return nil, err @@ -288,7 +288,7 @@ func (opts *Remote) NewRepository(reference string, log func(...interface{}), co once := sync.Once{} repo.HandleWarning = func(warning remote.Warning) { once.Do(func() { - log(warning.Text) + warn(warning.Text) }) } } diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index c94baf37c..1e3c4e953 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -111,7 +111,7 @@ func parseOCILayoutReference(raw string) (path string, ref string, err error) { } // NewTarget generates a new target based on opts. -func (opts *Target) NewTarget(common Common, log func(...interface{})) (oras.GraphTarget, error) { +func (opts *Target) NewTarget(common Common, warn func(...interface{})) (oras.GraphTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index e6590d810..41361994c 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -93,7 +93,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder } func runAttach(ctx context.Context, opts attachOptions) error { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) annotations, err := opts.LoadManifestAnnotations() if err != nil { return err diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 7b60c522c..64f1120ee 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -96,7 +96,7 @@ Example - Copy an artifact with multiple tags with concurrency tuned: } func runCopy(ctx context.Context, opts copyOptions) error { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) // Prepare source src, err := opts.From.NewReadonlyTarget(ctx, opts.Common) diff --git a/cmd/oras/root/manifest/push.go b/cmd/oras/root/manifest/push.go index 0e77d03cb..ec6a51067 100644 --- a/cmd/oras/root/manifest/push.go +++ b/cmd/oras/root/manifest/push.go @@ -104,7 +104,7 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit } func pushManifest(ctx context.Context, opts pushOptions) error { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) var target oras.Target var err error target, err = opts.NewTarget(opts.Common, logger.Warn) From 399265d1f52a386e66160602c21b5965efa4a3f8 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 02:31:29 +0000 Subject: [PATCH 20/44] fix lint Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 8483e13ce..61ffac3c6 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -257,7 +257,7 @@ func (opts *Remote) NewRegistry(hostname string, warn func(...interface{}), comm hostname = reg.Reference.Registry reg.PlainHTTP = opts.isPlainHttp(hostname) - if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI != false { + if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI { once := sync.Once{} reg.HandleWarning = func(warning remote.Warning) { once.Do(func() { From 5413dd3e12b3e8e078b45acc067782371a5e24c0 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 02:32:57 +0000 Subject: [PATCH 21/44] fix lint Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 61ffac3c6..7d839f0cb 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -284,7 +284,7 @@ func (opts *Remote) NewRepository(reference string, warn func(...interface{}), c return nil, err } - if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI != false { + if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI { once := sync.Once{} repo.HandleWarning = func(warning remote.Warning) { once.Do(func() { From 177a19ac3328054b94c01962b34e2738af9407f5 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 02:54:54 +0000 Subject: [PATCH 22/44] bug fix Signed-off-by: Billy Zha --- cmd/oras/internal/option/target.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index 1e3c4e953..066388ad9 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -121,7 +121,7 @@ func (opts *Target) NewTarget(common Common, warn func(...interface{})) (oras.Gr } return oci.New(opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, nil, common) + repo, err := opts.NewRepository(opts.RawReference, warn, common) if err != nil { return nil, err } From 7cc3bf80acf5e996da42a3bcc9e2d7ea1e6aa29e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 16:06:58 +0000 Subject: [PATCH 23/44] add index support Signed-off-by: Billy Zha --- internal/graph/graph.go | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 77c054a59..bd6fe09ab 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -83,6 +83,19 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr } nodes = manifest.Blobs subject = manifest.Subject + + case ocispec.MediaTypeImageIndex: + var fetched []byte + fetched, err = content.FetchAll(ctx, fetcher, node) + if err != nil { + return + } + var index ocispec.Index + if err = json.Unmarshal(fetched, &index); err != nil { + return + } + nodes = index.Manifests + subject = index.Subject default: nodes, err = content.Successors(ctx, fetcher, node) } @@ -137,8 +150,25 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc if image.Subject == nil || !content.Equal(*image.Subject, desc) { continue } - node.ArtifactType = image.Config.MediaType + node.ArtifactType = image.ArtifactType + if node.ArtifactType == "" { + node.ArtifactType = image.Config.MediaType + } node.Annotations = image.Annotations + case ocispec.MediaTypeImageIndex: + fetched, err := fetchBytes(ctx, target, node) + if err != nil { + return nil, err + } + var index ocispec.Index + if err := json.Unmarshal(fetched, &index); err != nil { + return nil, err + } + if index.Subject == nil || !content.Equal(*index.Subject, desc) { + continue + } + node.ArtifactType = index.ArtifactType + node.Annotations = index.Annotations default: continue } From 788398b4b96bfcb857e951a5ddcb13967316556c Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 16:07:56 +0000 Subject: [PATCH 24/44] resolve comment Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index 988fa8e02..aa08fcae6 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -23,8 +23,8 @@ import ( ) const ( - v1_1 = "v1.1" - v1_0 = "v1.0" + ImageSpecV1_1 = "v1.1" + ImageSpecV1_0 = "v1.0" // TODO: pending on https://github.com/oras-project/oras-go/issues/568 PackManifestTypeImageV1_0 = 0 ) @@ -38,9 +38,9 @@ type ImageSpec struct { // Parse parses flags into the option. func (opts *ImageSpec) Parse() error { switch opts.flag { - case v1_1: + case ImageSpecV1_1: opts.PackType = oras.PackManifestTypeImageV1_1_0_RC4 - case v1_0: + case ImageSpecV1_0: opts.PackType = PackManifestTypeImageV1_0 default: return fmt.Errorf("unknown image specification flag: %q", opts.flag) @@ -50,7 +50,7 @@ func (opts *ImageSpec) Parse() error { // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, v1_0)) + fs.StringVar(&opts.flag, "image-spec", ImageSpecV1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", ImageSpecV1_1, ImageSpecV1_0)) } // distributionSpec option struct. From 93d7a8a4e580a8a99d8339c799e1e3739db4218e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 16:31:52 +0000 Subject: [PATCH 25/44] remove duplicated function Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 5 +- internal/graph/graph.go | 40 --------------- internal/graph/graph_test.go | 97 ------------------------------------ 3 files changed, 4 insertions(+), 138 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index f8a953b39..224de53f7 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -25,6 +25,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" "oras.land/oras/internal/graph" @@ -117,7 +118,9 @@ func runCopy(ctx context.Context, opts copyOptions) error { committed := &sync.Map{} extendedCopyOptions := oras.DefaultExtendedCopyOptions extendedCopyOptions.Concurrency = opts.concurrency - extendedCopyOptions.FindPredecessors = graph.FindReferrerPredecessors + extendedCopyOptions.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + return graph.Referrers(ctx, src, desc, "") + } extendedCopyOptions.PreCopy = display.StatusPrinter("Copying", opts.Verbose) extendedCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index bd6fe09ab..086a51447 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -179,46 +179,6 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc return results, nil } -// FindReferrerPredecessors returns referrer nodes of desc in target. -func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - var results []ocispec.Descriptor - if repo, ok := src.(registry.ReferrerLister); ok { - // get referrers directly - err := repo.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error { - results = append(results, referrers...) - return nil - }) - if err != nil { - return nil, err - } - return results, nil - } - predecessors, err := src.Predecessors(ctx, desc) - if err != nil { - return nil, err - } - for _, node := range predecessors { - switch node.MediaType { - case MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: - results = append(results, node) - case ocispec.MediaTypeImageIndex: - fetched, err := fetchBytes(ctx, src, node) - if err != nil { - return nil, err - } - // convert to json - var index ocispec.Index - if err := json.Unmarshal(fetched, &index); err != nil { - return nil, err - } - if index.Subject != nil && content.Equal(*index.Subject, desc) { - results = append(results, node) - } - } - } - return results, nil -} - func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]byte, error) { rc, err := fetcher.Fetch(ctx, desc) if err != nil { diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 370d34bb2..d33c4ad8c 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -39,14 +39,6 @@ func (e *errLister) Referrers(ctx context.Context, desc ocispec.Descriptor, arti return errors.New("") } -type errFinder struct { - oras.ReadOnlyGraphTarget -} - -func (e *errFinder) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { - return nil, errors.New("") -} - type refLister struct { referrers []ocispec.Descriptor oras.ReadOnlyGraphTarget @@ -265,92 +257,3 @@ func TestSuccessors(t *testing.T) { }) } } - -func TestFindReferrerPredecessors(t *testing.T) { - ctx := context.Background() - var blobs [][]byte - var descs []ocispec.Descriptor - appendBlob := func(mediaType string, blob []byte) { - blobs = append(blobs, blob) - descs = append(descs, ocispec.Descriptor{ - MediaType: mediaType, - Digest: digest.FromBytes(blob), - Size: int64(len(blob)), - }) - } - generateImage := func(subject *ocispec.Descriptor, annotations map[string]string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { - manifest := ocispec.Manifest{ - Subject: subject, - Config: config, - Layers: layers, - Annotations: annotations, - } - manifestJSON, err := json.Marshal(manifest) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) - } - generateIndex := func(manifests ...ocispec.Descriptor) { - index := ocispec.Index{ - Manifests: manifests, - } - manifestJSON, err := json.Marshal(index) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeImageIndex, manifestJSON) - } - const ( - subject = iota - imgConfig - image - ) - var anno map[string]string - appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) - imageType := "test.image" - appendBlob(imageType, []byte("config content")) - generateImage(&descs[subject], anno, descs[imgConfig]) - imageDesc := descs[image] - imageDesc.Annotations = anno - imageDesc.ArtifactType = imageType - generateIndex(descs[subject]) - - referrers := []ocispec.Descriptor{descs[image], descs[image]} - memory := memory.New() - for i := range descs { - if err := memory.Push(ctx, descs[i], bytes.NewReader(blobs[i])); err != nil { - t.Errorf("Error pushing %v\n", err) - } - } - finder := &predecessorFinder{Store: memory} - type args struct { - ctx context.Context - src content.ReadOnlyGraphStorage - desc ocispec.Descriptor - } - tests := []struct { - name string - args args - want []ocispec.Descriptor - wantErr bool - }{ - {"should failed to get referrers", args{ctx, &errLister{}, ocispec.Descriptor{}}, nil, true}, - {"should failed to get predecessor", args{ctx, &errFinder{}, ocispec.Descriptor{}}, nil, true}, - {"should return referrers when target is a referrer lister", args{ctx, &refLister{referrers: referrers}, ocispec.Descriptor{}}, referrers, false}, - {"should return image for config node", args{ctx, finder, descs[imgConfig]}, []ocispec.Descriptor{descs[image]}, false}, - {"should return image for subject node", args{ctx, finder, descs[subject]}, []ocispec.Descriptor{descs[image]}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := FindReferrerPredecessors(tt.args.ctx, tt.args.src, tt.args.desc) - if (err != nil) != tt.wantErr { - t.Errorf("FindReferrerPredecessors() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("FindReferrerPredecessors() = %v, want %v", got, tt.want) - } - }) - } -} From db9a803b15ed478375304bd367095d4a8344fa39 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 18 Aug 2023 05:44:10 +0000 Subject: [PATCH 26/44] fix `blob delete` and `login` Signed-off-by: Billy Zha --- cmd/oras/root/blob/delete.go | 4 ++-- cmd/oras/root/login.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/oras/root/blob/delete.go b/cmd/oras/root/blob/delete.go index 7e9d46bf8..95b50130a 100644 --- a/cmd/oras/root/blob/delete.go +++ b/cmd/oras/root/blob/delete.go @@ -71,8 +71,8 @@ Example - Delete a blob and print its descriptor: } func deleteBlob(ctx context.Context, opts deleteBlobOptions) (err error) { - ctx, _ = opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, nil, opts.Common) + ctx, logger := opts.WithContext(ctx) + repo, err := opts.NewRepository(opts.targetRef, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/login.go b/cmd/oras/root/login.go index 7bed32686..a3a0b8dc7 100644 --- a/cmd/oras/root/login.go +++ b/cmd/oras/root/login.go @@ -75,7 +75,7 @@ Example - Log in with username and password in an interactive terminal and no TL } func runLogin(ctx context.Context, opts loginOptions) (err error) { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) // prompt for credential if opts.Password == "" { @@ -108,7 +108,7 @@ func runLogin(ctx context.Context, opts loginOptions) (err error) { if err != nil { return err } - remote, err := opts.Remote.NewRegistry(opts.Hostname, nil, opts.Common) + remote, err := opts.Remote.NewRegistry(opts.Hostname, logger.Warn, opts.Common) if err != nil { return err } From 72b5f059a42a26696ad6ea3c5d36eeb744133e5b Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 18 Aug 2023 05:53:30 +0000 Subject: [PATCH 27/44] add warning to readonly targets Signed-off-by: Billy Zha --- cmd/oras/internal/option/target.go | 4 ++-- cmd/oras/root/blob/fetch.go | 4 ++-- cmd/oras/root/cp.go | 2 +- cmd/oras/root/discover.go | 4 ++-- cmd/oras/root/manifest/fetch.go | 4 ++-- cmd/oras/root/manifest/fetch_config.go | 4 ++-- cmd/oras/root/pull.go | 4 ++-- cmd/oras/root/repo/tags.go | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index 066388ad9..f383a2760 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -142,7 +142,7 @@ type ReadOnlyGraphTagFinderTarget interface { } // NewReadonlyTargets generates a new read only target based on opts. -func (opts *Target) NewReadonlyTarget(ctx context.Context, common Common) (ReadOnlyGraphTagFinderTarget, error) { +func (opts *Target) NewReadonlyTarget(ctx context.Context, warn func(...interface{}), common Common) (ReadOnlyGraphTagFinderTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error @@ -159,7 +159,7 @@ func (opts *Target) NewReadonlyTarget(ctx context.Context, common Common) (ReadO } return oci.NewFromTar(ctx, opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, nil, common) + repo, err := opts.NewRepository(opts.RawReference, warn, common) if err != nil { return nil, err } diff --git a/cmd/oras/root/blob/fetch.go b/cmd/oras/root/blob/fetch.go index 7f53873a4..307a27df0 100644 --- a/cmd/oras/root/blob/fetch.go +++ b/cmd/oras/root/blob/fetch.go @@ -90,9 +90,9 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' } func fetchBlob(ctx context.Context, opts fetchBlobOptions) (fetchErr error) { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) var target oras.ReadOnlyTarget - target, err := opts.NewReadonlyTarget(ctx, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 3d575640e..d4bb499bf 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -100,7 +100,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { ctx, logger := opts.WithContext(ctx) // Prepare source - src, err := opts.From.NewReadonlyTarget(ctx, opts.Common) + src, err := opts.From.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index b7dd372ae..9ef54008a 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -91,8 +91,8 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout } func runDiscover(ctx context.Context, opts discoverOptions) error { - ctx, _ = opts.WithContext(ctx) - repo, err := opts.NewReadonlyTarget(ctx, opts.Common) + ctx, logger := opts.WithContext(ctx) + repo, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/fetch.go b/cmd/oras/root/manifest/fetch.go index 850ac915f..eede5f45b 100644 --- a/cmd/oras/root/manifest/fetch.go +++ b/cmd/oras/root/manifest/fetch.go @@ -90,9 +90,9 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': } func fetchManifest(ctx context.Context, opts fetchOptions) (fetchErr error) { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) - target, err := opts.NewReadonlyTarget(ctx, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/fetch_config.go b/cmd/oras/root/manifest/fetch_config.go index a909a4d38..d342e3403 100644 --- a/cmd/oras/root/manifest/fetch_config.go +++ b/cmd/oras/root/manifest/fetch_config.go @@ -86,9 +86,9 @@ Example - Fetch and print the prettified descriptor of the config: } func fetchConfig(ctx context.Context, opts fetchConfigOptions) (fetchErr error) { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewReadonlyTarget(ctx, opts.Common) + repo, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/pull.go b/cmd/oras/root/pull.go index c10f330a4..3eab79426 100644 --- a/cmd/oras/root/pull.go +++ b/cmd/oras/root/pull.go @@ -103,7 +103,7 @@ Example - Pull artifact files from an OCI layout archive 'layout.tar': } func runPull(ctx context.Context, opts pullOptions) error { - ctx, _ = opts.WithContext(ctx) + ctx, logger := opts.WithContext(ctx) // Copy Options var printed sync.Map copyOptions := oras.DefaultCopyOptions @@ -182,7 +182,7 @@ func runPull(ctx context.Context, opts pullOptions) error { return ret, nil } - target, err := opts.NewReadonlyTarget(ctx, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/repo/tags.go b/cmd/oras/root/repo/tags.go index 2e7bde777..8df4d1c51 100644 --- a/cmd/oras/root/repo/tags.go +++ b/cmd/oras/root/repo/tags.go @@ -79,7 +79,7 @@ Example - [Experimental] Show tags associated with a digest: func showTags(ctx context.Context, opts showTagsOptions) error { ctx, logger := opts.WithContext(ctx) - finder, err := opts.NewReadonlyTarget(ctx, opts.Common) + finder, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { return err } From d11b379d78c2d28f495606a6bdf4282aa568a77a Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 18 Aug 2023 08:21:27 +0000 Subject: [PATCH 28/44] deduplicated based on warning value Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 7d839f0cb..a2c75578c 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -54,6 +54,7 @@ type Remote struct { distributionSpec distributionSpec headerFlags []string headers http.Header + hasWarned sync.Map } // EnableDistributionSpecFlag set distribution specification flag as applicable. @@ -257,12 +258,9 @@ func (opts *Remote) NewRegistry(hostname string, warn func(...interface{}), comm hostname = reg.Reference.Registry reg.PlainHTTP = opts.isPlainHttp(hostname) - if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI { - once := sync.Once{} - reg.HandleWarning = func(warning remote.Warning) { - once.Do(func() { - warn(warning.Text) - }) + reg.HandleWarning = func(warning remote.Warning) { + if _, ok := opts.hasWarned.LoadOrStore(warning.WarningValue, true); ok { + warn(warning.Text) } } if reg.Client, err = opts.authClient(hostname, common.Debug); err != nil { @@ -283,16 +281,11 @@ func (opts *Remote) NewRepository(reference string, warn func(...interface{}), c if repo.Client, err = opts.authClient(hostname, common.Debug); err != nil { return nil, err } - - if opts.distributionSpec.referrersAPI == nil || *opts.distributionSpec.referrersAPI { - once := sync.Once{} - repo.HandleWarning = func(warning remote.Warning) { - once.Do(func() { - warn(warning.Text) - }) + repo.HandleWarning = func(warning remote.Warning) { + if _, loaded := opts.hasWarned.LoadOrStore(warning.WarningValue, true); loaded { + warn(hostname, warning.Text) } } - if opts.distributionSpec.referrersAPI != nil { if err := repo.SetReferrersCapability(*opts.distributionSpec.referrersAPI); err != nil { return nil, err From 87de8dfb2dcdcbd29b9fdda2b83a661f5e35eebd Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 18 Aug 2023 08:59:59 +0000 Subject: [PATCH 29/44] udpate options to avoid copylocks error Signed-off-by: Billy Zha --- cmd/oras/root/attach.go | 4 ++-- cmd/oras/root/blob/delete.go | 4 ++-- cmd/oras/root/blob/fetch.go | 4 ++-- cmd/oras/root/blob/push.go | 4 ++-- cmd/oras/root/cp.go | 4 ++-- cmd/oras/root/discover.go | 8 ++++---- cmd/oras/root/login.go | 4 ++-- cmd/oras/root/logout.go | 4 ++-- cmd/oras/root/manifest/delete.go | 4 ++-- cmd/oras/root/manifest/fetch.go | 4 ++-- cmd/oras/root/manifest/fetch_config.go | 4 ++-- cmd/oras/root/manifest/push.go | 4 ++-- cmd/oras/root/pull.go | 4 ++-- cmd/oras/root/push.go | 4 ++-- cmd/oras/root/repo/ls.go | 4 ++-- cmd/oras/root/repo/tags.go | 4 ++-- cmd/oras/root/tag.go | 4 ++-- 17 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index 41361994c..2e07819a3 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -80,7 +80,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder return nil }, RunE: func(cmd *cobra.Command, args []string) error { - return runAttach(cmd.Context(), opts) + return runAttach(cmd.Context(), &opts) }, } @@ -92,7 +92,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder return cmd } -func runAttach(ctx context.Context, opts attachOptions) error { +func runAttach(ctx context.Context, opts *attachOptions) error { ctx, logger := opts.WithContext(ctx) annotations, err := opts.LoadManifestAnnotations() if err != nil { diff --git a/cmd/oras/root/blob/delete.go b/cmd/oras/root/blob/delete.go index 95b50130a..3cd40c972 100644 --- a/cmd/oras/root/blob/delete.go +++ b/cmd/oras/root/blob/delete.go @@ -62,7 +62,7 @@ Example - Delete a blob and print its descriptor: }, RunE: func(cmd *cobra.Command, args []string) error { opts.targetRef = args[0] - return deleteBlob(cmd.Context(), opts) + return deleteBlob(cmd.Context(), &opts) }, } @@ -70,7 +70,7 @@ Example - Delete a blob and print its descriptor: return cmd } -func deleteBlob(ctx context.Context, opts deleteBlobOptions) (err error) { +func deleteBlob(ctx context.Context, opts *deleteBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewRepository(opts.targetRef, logger.Warn, opts.Common) if err != nil { diff --git a/cmd/oras/root/blob/fetch.go b/cmd/oras/root/blob/fetch.go index 307a27df0..b9cc6ac24 100644 --- a/cmd/oras/root/blob/fetch.go +++ b/cmd/oras/root/blob/fetch.go @@ -80,7 +80,7 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' }, Aliases: []string{"get"}, RunE: func(cmd *cobra.Command, args []string) error { - return fetchBlob(cmd.Context(), opts) + return fetchBlob(cmd.Context(), &opts) }, } @@ -89,7 +89,7 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' return cmd } -func fetchBlob(ctx context.Context, opts fetchBlobOptions) (fetchErr error) { +func fetchBlob(ctx context.Context, opts *fetchBlobOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) var target oras.ReadOnlyTarget target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) diff --git a/cmd/oras/root/blob/push.go b/cmd/oras/root/blob/push.go index f973042a4..cd6ddedfb 100644 --- a/cmd/oras/root/blob/push.go +++ b/cmd/oras/root/blob/push.go @@ -85,7 +85,7 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir': return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return pushBlob(cmd.Context(), opts) + return pushBlob(cmd.Context(), &opts) }, } @@ -95,7 +95,7 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir': return cmd } -func pushBlob(ctx context.Context, opts pushBlobOptions) (err error) { +func pushBlob(ctx context.Context, opts *pushBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewTarget(opts.Common, logger.Warn) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index d4bb499bf..1fe212d0d 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -86,7 +86,7 @@ Example - Copy an artifact with multiple tags with concurrency tuned: return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return runCopy(cmd.Context(), opts) + return runCopy(cmd.Context(), &opts) }, } cmd.Flags().BoolVarP(&opts.recursive, "recursive", "r", false, "[Preview] recursively copy the artifact and its referrer artifacts") @@ -96,7 +96,7 @@ Example - Copy an artifact with multiple tags with concurrency tuned: return cmd } -func runCopy(ctx context.Context, opts copyOptions) error { +func runCopy(ctx context.Context, opts *copyOptions) error { ctx, logger := opts.WithContext(ctx) // Prepare source diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index 9ef54008a..fbda1670c 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -79,7 +79,7 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return runDiscover(cmd.Context(), opts) + return runDiscover(cmd.Context(), &opts) }, } @@ -90,7 +90,7 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout return cmd } -func runDiscover(ctx context.Context, opts discoverOptions) error { +func runDiscover(ctx context.Context, opts *discoverOptions) error { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { @@ -110,7 +110,7 @@ func runDiscover(ctx context.Context, opts discoverOptions) error { if opts.outputType == "tree" { root := tree.New(fmt.Sprintf("%s@%s", opts.Path, desc.Digest)) - err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, &opts) + err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, opts) if err != nil { return err } @@ -189,7 +189,7 @@ func printDiscoveredReferrersTable(refs []ocispec.Descriptor, verbose bool) erro print(ref.ArtifactType, ref.Digest) if verbose { if err := printJSON(ref); err != nil { - return fmt.Errorf("Error printing JSON: %w", err) + return fmt.Errorf("Eirror printing JSON: %w", err) } } } diff --git a/cmd/oras/root/login.go b/cmd/oras/root/login.go index a3a0b8dc7..cb2dcbf77 100644 --- a/cmd/oras/root/login.go +++ b/cmd/oras/root/login.go @@ -67,14 +67,14 @@ Example - Log in with username and password in an interactive terminal and no TL }, RunE: func(cmd *cobra.Command, args []string) error { opts.Hostname = args[0] - return runLogin(cmd.Context(), opts) + return runLogin(cmd.Context(), &opts) }, } option.ApplyFlags(&opts, cmd.Flags()) return cmd } -func runLogin(ctx context.Context, opts loginOptions) (err error) { +func runLogin(ctx context.Context, opts *loginOptions) (err error) { ctx, logger := opts.WithContext(ctx) // prompt for credential diff --git a/cmd/oras/root/logout.go b/cmd/oras/root/logout.go index bdb39d1f5..b4d4585ec 100644 --- a/cmd/oras/root/logout.go +++ b/cmd/oras/root/logout.go @@ -44,7 +44,7 @@ Example - Logout: Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { opts.hostname = args[0] - return runLogout(cmd.Context(), opts) + return runLogout(cmd.Context(), &opts) }, } @@ -53,7 +53,7 @@ Example - Logout: return cmd } -func runLogout(ctx context.Context, opts logoutOptions) error { +func runLogout(ctx context.Context, opts *logoutOptions) error { if opts.debug { logrus.SetLevel(logrus.DebugLevel) } diff --git a/cmd/oras/root/manifest/delete.go b/cmd/oras/root/manifest/delete.go index 1aae58a64..5ca31cbad 100644 --- a/cmd/oras/root/manifest/delete.go +++ b/cmd/oras/root/manifest/delete.go @@ -66,7 +66,7 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 }, RunE: func(cmd *cobra.Command, args []string) error { opts.targetRef = args[0] - return deleteManifest(cmd.Context(), opts) + return deleteManifest(cmd.Context(), &opts) }, } @@ -75,7 +75,7 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 return cmd } -func deleteManifest(ctx context.Context, opts deleteOptions) error { +func deleteManifest(ctx context.Context, opts *deleteOptions) error { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewRepository(opts.targetRef, logger.Warn, opts.Common) if err != nil { diff --git a/cmd/oras/root/manifest/fetch.go b/cmd/oras/root/manifest/fetch.go index eede5f45b..f796d9995 100644 --- a/cmd/oras/root/manifest/fetch.go +++ b/cmd/oras/root/manifest/fetch.go @@ -79,7 +79,7 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': }, Aliases: []string{"get"}, RunE: func(cmd *cobra.Command, args []string) error { - return fetchManifest(cmd.Context(), opts) + return fetchManifest(cmd.Context(), &opts) }, } @@ -89,7 +89,7 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': return cmd } -func fetchManifest(ctx context.Context, opts fetchOptions) (fetchErr error) { +func fetchManifest(ctx context.Context, opts *fetchOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) diff --git a/cmd/oras/root/manifest/fetch_config.go b/cmd/oras/root/manifest/fetch_config.go index d342e3403..c769b52b0 100644 --- a/cmd/oras/root/manifest/fetch_config.go +++ b/cmd/oras/root/manifest/fetch_config.go @@ -76,7 +76,7 @@ Example - Fetch and print the prettified descriptor of the config: return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return fetchConfig(cmd.Context(), opts) + return fetchConfig(cmd.Context(), &opts) }, } @@ -85,7 +85,7 @@ Example - Fetch and print the prettified descriptor of the config: return cmd } -func fetchConfig(ctx context.Context, opts fetchConfigOptions) (fetchErr error) { +func fetchConfig(ctx context.Context, opts *fetchConfigOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) diff --git a/cmd/oras/root/manifest/push.go b/cmd/oras/root/manifest/push.go index ec6a51067..e2a1edc1c 100644 --- a/cmd/oras/root/manifest/push.go +++ b/cmd/oras/root/manifest/push.go @@ -92,7 +92,7 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return pushManifest(cmd.Context(), opts) + return pushManifest(cmd.Context(), &opts) }, } @@ -103,7 +103,7 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit return cmd } -func pushManifest(ctx context.Context, opts pushOptions) error { +func pushManifest(ctx context.Context, opts *pushOptions) error { ctx, logger := opts.WithContext(ctx) var target oras.Target var err error diff --git a/cmd/oras/root/pull.go b/cmd/oras/root/pull.go index 3eab79426..7576135a4 100644 --- a/cmd/oras/root/pull.go +++ b/cmd/oras/root/pull.go @@ -88,7 +88,7 @@ Example - Pull artifact files from an OCI layout archive 'layout.tar': return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return runPull(cmd.Context(), opts) + return runPull(cmd.Context(), &opts) }, } @@ -102,7 +102,7 @@ Example - Pull artifact files from an OCI layout archive 'layout.tar': return cmd } -func runPull(ctx context.Context, opts pullOptions) error { +func runPull(ctx context.Context, opts *pullOptions) error { ctx, logger := opts.WithContext(ctx) // Copy Options var printed sync.Map diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 1343e62ba..99058a3aa 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -114,7 +114,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return nil }, RunE: func(cmd *cobra.Command, args []string) error { - return runPush(cmd.Context(), opts) + return runPush(cmd.Context(), &opts) }, } cmd.Flags().StringVarP(&opts.manifestConfigRef, "config", "", "", "`path` of image config file") @@ -125,7 +125,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return cmd } -func runPush(ctx context.Context, opts pushOptions) error { +func runPush(ctx context.Context, opts *pushOptions) error { ctx, logger := opts.WithContext(ctx) annotations, err := opts.LoadManifestAnnotations() if err != nil { diff --git a/cmd/oras/root/repo/ls.go b/cmd/oras/root/repo/ls.go index e72298401..ee17269e0 100644 --- a/cmd/oras/root/repo/ls.go +++ b/cmd/oras/root/repo/ls.go @@ -59,7 +59,7 @@ Example - List the repositories under the registry that include values lexically if opts.hostname, opts.namespace, err = repository.ParseRepoPath(args[0]); err != nil { return fmt.Errorf("could not parse repository path: %w", err) } - return listRepository(cmd.Context(), opts) + return listRepository(cmd.Context(), &opts) }, } @@ -68,7 +68,7 @@ Example - List the repositories under the registry that include values lexically return cmd } -func listRepository(ctx context.Context, opts repositoryOptions) error { +func listRepository(ctx context.Context, opts *repositoryOptions) error { ctx, logger := opts.WithContext(ctx) reg, err := opts.Remote.NewRegistry(opts.hostname, logger.Warn, opts.Common) if err != nil { diff --git a/cmd/oras/root/repo/tags.go b/cmd/oras/root/repo/tags.go index 8df4d1c51..3c3f388e7 100644 --- a/cmd/oras/root/repo/tags.go +++ b/cmd/oras/root/repo/tags.go @@ -68,7 +68,7 @@ Example - [Experimental] Show tags associated with a digest: return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return showTags(cmd.Context(), opts) + return showTags(cmd.Context(), &opts) }, } cmd.Flags().StringVar(&opts.last, "last", "", "start after the tag specified by `last`") @@ -77,7 +77,7 @@ Example - [Experimental] Show tags associated with a digest: return cmd } -func showTags(ctx context.Context, opts showTagsOptions) error { +func showTags(ctx context.Context, opts *showTagsOptions) error { ctx, logger := opts.WithContext(ctx) finder, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) if err != nil { diff --git a/cmd/oras/root/tag.go b/cmd/oras/root/tag.go index 93c8a3722..772b2d276 100644 --- a/cmd/oras/root/tag.go +++ b/cmd/oras/root/tag.go @@ -66,7 +66,7 @@ Example - Tag the manifest 'v1.0.1' to 'v1.0.2' in an OCI image layout folder 'l return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return tagManifest(cmd.Context(), opts) + return tagManifest(cmd.Context(), &opts) }, } @@ -75,7 +75,7 @@ Example - Tag the manifest 'v1.0.1' to 'v1.0.2' in an OCI image layout folder 'l return cmd } -func tagManifest(ctx context.Context, opts tagOptions) error { +func tagManifest(ctx context.Context, opts *tagOptions) error { ctx, logger := opts.WithContext(ctx) target, err := opts.NewTarget(opts.Common, logger.Warn) if err != nil { From 6fcd346032e161e2aa05e235d7956d76306a8f15 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Mon, 21 Aug 2023 12:16:59 +0000 Subject: [PATCH 30/44] resolve comments Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index a2c75578c..ebc658d65 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -54,7 +54,7 @@ type Remote struct { distributionSpec distributionSpec headerFlags []string headers http.Header - hasWarned sync.Map + warned sync.Map } // EnableDistributionSpecFlag set distribution specification flag as applicable. @@ -250,7 +250,7 @@ func (opts *Remote) Credential() auth.Credential { } // NewRegistry assembles a oras remote registry. -func (opts *Remote) NewRegistry(hostname string, warn func(...interface{}), common Common) (reg *remote.Registry, err error) { +func (opts *Remote) NewRegistry(hostname string, warn func(...any), common Common) (reg *remote.Registry, err error) { reg, err = remote.NewRegistry(hostname) if err != nil { return nil, err @@ -259,7 +259,7 @@ func (opts *Remote) NewRegistry(hostname string, warn func(...interface{}), comm reg.PlainHTTP = opts.isPlainHttp(hostname) reg.HandleWarning = func(warning remote.Warning) { - if _, ok := opts.hasWarned.LoadOrStore(warning.WarningValue, true); ok { + if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { warn(warning.Text) } } @@ -270,7 +270,7 @@ func (opts *Remote) NewRegistry(hostname string, warn func(...interface{}), comm } // NewRepository assembles a oras remote repository. -func (opts *Remote) NewRepository(reference string, warn func(...interface{}), common Common) (repo *remote.Repository, err error) { +func (opts *Remote) NewRepository(reference string, warn func(...any), common Common) (repo *remote.Repository, err error) { repo, err = remote.NewRepository(reference) if err != nil { return nil, err @@ -282,7 +282,7 @@ func (opts *Remote) NewRepository(reference string, warn func(...interface{}), c return nil, err } repo.HandleWarning = func(warning remote.Warning) { - if _, loaded := opts.hasWarned.LoadOrStore(warning.WarningValue, true); loaded { + if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { warn(hostname, warning.Text) } } From aca52fff2d671198424e77074d618fcb85827f93 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 03:25:01 +0000 Subject: [PATCH 31/44] fix warning Signed-off-by: Billy Zha --- cmd/oras/internal/option/target.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index f383a2760..952862be2 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -111,7 +111,7 @@ func parseOCILayoutReference(raw string) (path string, ref string, err error) { } // NewTarget generates a new target based on opts. -func (opts *Target) NewTarget(common Common, warn func(...interface{})) (oras.GraphTarget, error) { +func (opts *Target) NewTarget(common Common, warn func(...any)) (oras.GraphTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error @@ -142,7 +142,7 @@ type ReadOnlyGraphTagFinderTarget interface { } // NewReadonlyTargets generates a new read only target based on opts. -func (opts *Target) NewReadonlyTarget(ctx context.Context, warn func(...interface{}), common Common) (ReadOnlyGraphTagFinderTarget, error) { +func (opts *Target) NewReadonlyTarget(ctx context.Context, warn func(...any), common Common) (ReadOnlyGraphTagFinderTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error From 0a044a20a034abdf1447502b6b6a03b484c8b760 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 15:07:59 +0800 Subject: [PATCH 32/44] fix windows test error Signed-off-by: Billy Zha --- internal/credential/store_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/credential/store_test.go b/internal/credential/store_test.go index c7f94569b..3a873ab14 100644 --- a/internal/credential/store_test.go +++ b/internal/credential/store_test.go @@ -26,10 +26,12 @@ import ( func TestNewStoreError(t *testing.T) { tmpDir := t.TempDir() filename := path.Join(tmpDir, "testfile.txt") - _, err := os.Create(filename) + file, err := os.Create(filename) if err != nil { t.Errorf("error: cannot create file : %v", err) } + defer file.Close() + err = os.Chmod(filename, 000) if err != nil { t.Errorf("error: cannot change file permissions: %v", err) From bbb05466e800605af13299d7cf313dfa542dada1 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 15:47:54 +0800 Subject: [PATCH 33/44] fix error Signed-off-by: Billy Zha --- cmd/oras/root/discover.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index fbda1670c..94c0de4a9 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -189,7 +189,7 @@ func printDiscoveredReferrersTable(refs []ocispec.Descriptor, verbose bool) erro print(ref.ArtifactType, ref.Digest) if verbose { if err := printJSON(ref); err != nil { - return fmt.Errorf("Eirror printing JSON: %w", err) + return fmt.Errorf("error printing JSON: %w", err) } } } From 3d3b8a7b844cdc05aa572eaead94e4d28ef46e30 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 17:20:22 +0800 Subject: [PATCH 34/44] add field to logger Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 10 ++++++---- cmd/oras/internal/option/target.go | 9 +++++---- cmd/oras/root/attach.go | 2 +- cmd/oras/root/blob/delete.go | 2 +- cmd/oras/root/blob/fetch.go | 2 +- cmd/oras/root/blob/push.go | 2 +- cmd/oras/root/cp.go | 4 ++-- cmd/oras/root/discover.go | 2 +- cmd/oras/root/login.go | 2 +- cmd/oras/root/manifest/delete.go | 2 +- cmd/oras/root/manifest/fetch.go | 2 +- cmd/oras/root/manifest/fetch_config.go | 2 +- cmd/oras/root/manifest/push.go | 2 +- cmd/oras/root/pull.go | 2 +- cmd/oras/root/push.go | 2 +- cmd/oras/root/repo/ls.go | 2 +- cmd/oras/root/repo/tags.go | 2 +- cmd/oras/root/tag.go | 2 +- 18 files changed, 28 insertions(+), 25 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index ebc658d65..873eb2980 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -28,6 +28,7 @@ import ( "sync" credentials "github.com/oras-project/oras-credentials-go" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" @@ -250,7 +251,7 @@ func (opts *Remote) Credential() auth.Credential { } // NewRegistry assembles a oras remote registry. -func (opts *Remote) NewRegistry(hostname string, warn func(...any), common Common) (reg *remote.Registry, err error) { +func (opts *Remote) NewRegistry(hostname string, logger logrus.FieldLogger, common Common) (reg *remote.Registry, err error) { reg, err = remote.NewRegistry(hostname) if err != nil { return nil, err @@ -260,7 +261,7 @@ func (opts *Remote) NewRegistry(hostname string, warn func(...any), common Commo reg.HandleWarning = func(warning remote.Warning) { if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { - warn(warning.Text) + logger.Warn(warning.Text) } } if reg.Client, err = opts.authClient(hostname, common.Debug); err != nil { @@ -270,7 +271,7 @@ func (opts *Remote) NewRegistry(hostname string, warn func(...any), common Commo } // NewRepository assembles a oras remote repository. -func (opts *Remote) NewRepository(reference string, warn func(...any), common Common) (repo *remote.Repository, err error) { +func (opts *Remote) NewRepository(reference string, logger logrus.FieldLogger, common Common) (repo *remote.Repository, err error) { repo, err = remote.NewRepository(reference) if err != nil { return nil, err @@ -281,9 +282,10 @@ func (opts *Remote) NewRepository(reference string, warn func(...any), common Co if repo.Client, err = opts.authClient(hostname, common.Debug); err != nil { return nil, err } + logger = logger.WithField("hostname", hostname) repo.HandleWarning = func(warning remote.Warning) { if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { - warn(hostname, warning.Text) + logger.Warn(warning.Text) } } if opts.distributionSpec.referrersAPI != nil { diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index 952862be2..df5e676ae 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -22,6 +22,7 @@ import ( "os" "strings" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/oci" @@ -111,7 +112,7 @@ func parseOCILayoutReference(raw string) (path string, ref string, err error) { } // NewTarget generates a new target based on opts. -func (opts *Target) NewTarget(common Common, warn func(...any)) (oras.GraphTarget, error) { +func (opts *Target) NewTarget(common Common, logger logrus.FieldLogger) (oras.GraphTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error @@ -121,7 +122,7 @@ func (opts *Target) NewTarget(common Common, warn func(...any)) (oras.GraphTarge } return oci.New(opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, warn, common) + repo, err := opts.NewRepository(opts.RawReference, logger, common) if err != nil { return nil, err } @@ -142,7 +143,7 @@ type ReadOnlyGraphTagFinderTarget interface { } // NewReadonlyTargets generates a new read only target based on opts. -func (opts *Target) NewReadonlyTarget(ctx context.Context, warn func(...any), common Common) (ReadOnlyGraphTagFinderTarget, error) { +func (opts *Target) NewReadonlyTarget(ctx context.Context, logger logrus.FieldLogger, common Common) (ReadOnlyGraphTagFinderTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error @@ -159,7 +160,7 @@ func (opts *Target) NewReadonlyTarget(ctx context.Context, warn func(...any), co } return oci.NewFromTar(ctx, opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, warn, common) + repo, err := opts.NewRepository(opts.RawReference, logger, common) if err != nil { return nil, err } diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index 2e07819a3..7cb897b5a 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -109,7 +109,7 @@ func runAttach(ctx context.Context, opts *attachOptions) error { } defer store.Close() - dst, err := opts.NewTarget(opts.Common, logger.Warn) + dst, err := opts.NewTarget(opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/blob/delete.go b/cmd/oras/root/blob/delete.go index 3cd40c972..c39dbc331 100644 --- a/cmd/oras/root/blob/delete.go +++ b/cmd/oras/root/blob/delete.go @@ -72,7 +72,7 @@ Example - Delete a blob and print its descriptor: func deleteBlob(ctx context.Context, opts *deleteBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, logger.Warn, opts.Common) + repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/blob/fetch.go b/cmd/oras/root/blob/fetch.go index b9cc6ac24..7915811a1 100644 --- a/cmd/oras/root/blob/fetch.go +++ b/cmd/oras/root/blob/fetch.go @@ -92,7 +92,7 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' func fetchBlob(ctx context.Context, opts *fetchBlobOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) var target oras.ReadOnlyTarget - target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/blob/push.go b/cmd/oras/root/blob/push.go index cd6ddedfb..2a060aeda 100644 --- a/cmd/oras/root/blob/push.go +++ b/cmd/oras/root/blob/push.go @@ -98,7 +98,7 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir': func pushBlob(ctx context.Context, opts *pushBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewTarget(opts.Common, logger.Warn) + repo, err := opts.NewTarget(opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 1fe212d0d..15aa1bac9 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -100,7 +100,7 @@ func runCopy(ctx context.Context, opts *copyOptions) error { ctx, logger := opts.WithContext(ctx) // Prepare source - src, err := opts.From.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + src, err := opts.From.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } @@ -109,7 +109,7 @@ func runCopy(ctx context.Context, opts *copyOptions) error { } // Prepare destination - dst, err := opts.To.NewTarget(opts.Common, logger.Warn) + dst, err := opts.To.NewTarget(opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index 94c0de4a9..63676d9d8 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -92,7 +92,7 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout func runDiscover(ctx context.Context, opts *discoverOptions) error { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/login.go b/cmd/oras/root/login.go index cb2dcbf77..52cca58ca 100644 --- a/cmd/oras/root/login.go +++ b/cmd/oras/root/login.go @@ -108,7 +108,7 @@ func runLogin(ctx context.Context, opts *loginOptions) (err error) { if err != nil { return err } - remote, err := opts.Remote.NewRegistry(opts.Hostname, logger.Warn, opts.Common) + remote, err := opts.Remote.NewRegistry(opts.Hostname, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/delete.go b/cmd/oras/root/manifest/delete.go index 5ca31cbad..9fd8b48f2 100644 --- a/cmd/oras/root/manifest/delete.go +++ b/cmd/oras/root/manifest/delete.go @@ -77,7 +77,7 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 func deleteManifest(ctx context.Context, opts *deleteOptions) error { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, logger.Warn, opts.Common) + repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/fetch.go b/cmd/oras/root/manifest/fetch.go index f796d9995..6d8111add 100644 --- a/cmd/oras/root/manifest/fetch.go +++ b/cmd/oras/root/manifest/fetch.go @@ -92,7 +92,7 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': func fetchManifest(ctx context.Context, opts *fetchOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) - target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/fetch_config.go b/cmd/oras/root/manifest/fetch_config.go index c769b52b0..41a2cddfc 100644 --- a/cmd/oras/root/manifest/fetch_config.go +++ b/cmd/oras/root/manifest/fetch_config.go @@ -88,7 +88,7 @@ Example - Fetch and print the prettified descriptor of the config: func fetchConfig(ctx context.Context, opts *fetchConfigOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/manifest/push.go b/cmd/oras/root/manifest/push.go index e2a1edc1c..ad1647f6c 100644 --- a/cmd/oras/root/manifest/push.go +++ b/cmd/oras/root/manifest/push.go @@ -107,7 +107,7 @@ func pushManifest(ctx context.Context, opts *pushOptions) error { ctx, logger := opts.WithContext(ctx) var target oras.Target var err error - target, err = opts.NewTarget(opts.Common, logger.Warn) + target, err = opts.NewTarget(opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/pull.go b/cmd/oras/root/pull.go index 7576135a4..7fd317cd4 100644 --- a/cmd/oras/root/pull.go +++ b/cmd/oras/root/pull.go @@ -182,7 +182,7 @@ func runPull(ctx context.Context, opts *pullOptions) error { return ret, nil } - target, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 99058a3aa..0768e54bc 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -173,7 +173,7 @@ func runPush(ctx context.Context, opts *pushOptions) error { } // prepare push - dst, err := opts.NewTarget(opts.Common, logger.Warn) + dst, err := opts.NewTarget(opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/repo/ls.go b/cmd/oras/root/repo/ls.go index ee17269e0..32031c722 100644 --- a/cmd/oras/root/repo/ls.go +++ b/cmd/oras/root/repo/ls.go @@ -70,7 +70,7 @@ Example - List the repositories under the registry that include values lexically func listRepository(ctx context.Context, opts *repositoryOptions) error { ctx, logger := opts.WithContext(ctx) - reg, err := opts.Remote.NewRegistry(opts.hostname, logger.Warn, opts.Common) + reg, err := opts.Remote.NewRegistry(opts.hostname, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/repo/tags.go b/cmd/oras/root/repo/tags.go index 3c3f388e7..ed98a757b 100644 --- a/cmd/oras/root/repo/tags.go +++ b/cmd/oras/root/repo/tags.go @@ -79,7 +79,7 @@ Example - [Experimental] Show tags associated with a digest: func showTags(ctx context.Context, opts *showTagsOptions) error { ctx, logger := opts.WithContext(ctx) - finder, err := opts.NewReadonlyTarget(ctx, logger.Warn, opts.Common) + finder, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { return err } diff --git a/cmd/oras/root/tag.go b/cmd/oras/root/tag.go index 772b2d276..28ec967d4 100644 --- a/cmd/oras/root/tag.go +++ b/cmd/oras/root/tag.go @@ -77,7 +77,7 @@ Example - Tag the manifest 'v1.0.1' to 'v1.0.2' in an OCI image layout folder 'l func tagManifest(ctx context.Context, opts *tagOptions) error { ctx, logger := opts.WithContext(ctx) - target, err := opts.NewTarget(opts.Common, logger.Warn) + target, err := opts.NewTarget(opts.Common, logger) if err != nil { return err } From 7b777eae39a05ce76bae8ed969377b84b558d0b5 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 17:22:04 +0800 Subject: [PATCH 35/44] nit Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 873eb2980..9cac51992 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -258,7 +258,6 @@ func (opts *Remote) NewRegistry(hostname string, logger logrus.FieldLogger, comm } hostname = reg.Reference.Registry reg.PlainHTTP = opts.isPlainHttp(hostname) - reg.HandleWarning = func(warning remote.Warning) { if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { logger.Warn(warning.Text) From 46220d6b154c28bbefa0fec0ff87fa67c2ef8332 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 14:21:52 +0000 Subject: [PATCH 36/44] fix unit test Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/oras/internal/option/remote_test.go b/cmd/oras/internal/option/remote_test.go index f7b105091..63881a0ba 100644 --- a/cmd/oras/internal/option/remote_test.go +++ b/cmd/oras/internal/option/remote_test.go @@ -30,6 +30,7 @@ import ( "reflect" "testing" + "github.com/sirupsen/logrus" "github.com/spf13/pflag" "oras.land/oras-go/v2/registry/remote/auth" ) @@ -180,7 +181,7 @@ func TestRemote_NewRegistry(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - reg, err := opts.NewRegistry(uri.Host, nil, opts.Common) + reg, err := opts.NewRegistry(uri.Host, logrus.New(), opts.Common) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -208,7 +209,7 @@ func TestRemote_NewRepository(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - repo, err := opts.NewRepository(uri.Host+"/"+testRepo, nil, opts.Common) + repo, err := opts.NewRepository(uri.Host+"/"+testRepo, logrus.New(), opts.Common) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -255,7 +256,7 @@ func TestRemote_NewRepository_Retry(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - repo, err := opts.NewRepository(uri.Host+"/"+testRepo, nil, opts.Common) + repo, err := opts.NewRepository(uri.Host+"/"+testRepo, logrus.New(), opts.Common) if err != nil { t.Fatalf("unexpected error: %v", err) } From 698f5fb1b3af4601d097c65bada8db93441a3b01 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 14:22:46 +0000 Subject: [PATCH 37/44] refactor and make warning depulicated via registry Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 40 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 9cac51992..85cc5a3ad 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -249,21 +249,28 @@ func (opts *Remote) parseCustomHeaders() error { func (opts *Remote) Credential() auth.Credential { return credential.Credential(opts.Username, opts.Password) } +func (opts *Remote) handleWarning(registry string, logger logrus.FieldLogger) func(warning remote.Warning) { + logger = logger.WithField("registry", registry) + return func(warning remote.Warning) { + if _, loaded := opts.warned.LoadOrStore(struct { + string + remote.WarningValue + }{registry, warning.WarningValue}, true); !loaded { + logger.Warn(warning.Text) + } + } +} // NewRegistry assembles a oras remote registry. -func (opts *Remote) NewRegistry(hostname string, logger logrus.FieldLogger, common Common) (reg *remote.Registry, err error) { - reg, err = remote.NewRegistry(hostname) +func (opts *Remote) NewRegistry(registry string, logger logrus.FieldLogger, common Common) (reg *remote.Registry, err error) { + reg, err = remote.NewRegistry(registry) if err != nil { return nil, err } - hostname = reg.Reference.Registry - reg.PlainHTTP = opts.isPlainHttp(hostname) - reg.HandleWarning = func(warning remote.Warning) { - if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { - logger.Warn(warning.Text) - } - } - if reg.Client, err = opts.authClient(hostname, common.Debug); err != nil { + registry = reg.Reference.Registry + reg.PlainHTTP = opts.isPlainHttp(registry) + reg.HandleWarning = opts.handleWarning(registry, logger) + if reg.Client, err = opts.authClient(registry, common.Debug); err != nil { return nil, err } return @@ -275,18 +282,13 @@ func (opts *Remote) NewRepository(reference string, logger logrus.FieldLogger, c if err != nil { return nil, err } - hostname := repo.Reference.Registry - repo.PlainHTTP = opts.isPlainHttp(hostname) + registry := repo.Reference.Registry + repo.PlainHTTP = opts.isPlainHttp(registry) repo.SkipReferrersGC = true - if repo.Client, err = opts.authClient(hostname, common.Debug); err != nil { + if repo.Client, err = opts.authClient(registry, common.Debug); err != nil { return nil, err } - logger = logger.WithField("hostname", hostname) - repo.HandleWarning = func(warning remote.Warning) { - if _, loaded := opts.warned.LoadOrStore(warning.WarningValue, true); !loaded { - logger.Warn(warning.Text) - } - } + repo.HandleWarning = opts.handleWarning(registry, logger) if opts.distributionSpec.referrersAPI != nil { if err := repo.SetReferrersCapability(*opts.distributionSpec.referrersAPI); err != nil { return nil, err From 11c46b756f850d513ceb29cc8709ea06d64a4487 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 14:37:10 +0000 Subject: [PATCH 38/44] make map a point and init when parsing Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 3 ++- cmd/oras/root/attach.go | 4 ++-- cmd/oras/root/blob/delete.go | 4 ++-- cmd/oras/root/blob/fetch.go | 4 ++-- cmd/oras/root/blob/push.go | 4 ++-- cmd/oras/root/cp.go | 4 ++-- cmd/oras/root/discover.go | 12 ++++++------ cmd/oras/root/login.go | 4 ++-- cmd/oras/root/logout.go | 4 ++-- cmd/oras/root/manifest/delete.go | 4 ++-- cmd/oras/root/manifest/fetch.go | 4 ++-- cmd/oras/root/manifest/fetch_config.go | 4 ++-- cmd/oras/root/manifest/push.go | 4 ++-- cmd/oras/root/pull.go | 4 ++-- cmd/oras/root/push.go | 4 ++-- cmd/oras/root/repo/ls.go | 4 ++-- cmd/oras/root/repo/tags.go | 4 ++-- cmd/oras/root/tag.go | 4 ++-- 18 files changed, 40 insertions(+), 39 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 85cc5a3ad..927780f9d 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -55,7 +55,7 @@ type Remote struct { distributionSpec distributionSpec headerFlags []string headers http.Header - warned sync.Map + warned *sync.Map } // EnableDistributionSpecFlag set distribution specification flag as applicable. @@ -107,6 +107,7 @@ func (opts *Remote) ApplyFlagsWithPrefix(fs *pflag.FlagSet, prefix, description // Parse tries to read password with optional cmd prompt. func (opts *Remote) Parse() error { + opts.warned = &sync.Map{} if err := opts.parseCustomHeaders(); err != nil { return err } diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index 4358a0841..b67ca0c99 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -80,7 +80,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder return nil }, RunE: func(cmd *cobra.Command, args []string) error { - return runAttach(cmd.Context(), &opts) + return runAttach(cmd.Context(), opts) }, } @@ -92,7 +92,7 @@ Example - Attach file to the manifest tagged 'v1' in an OCI image layout folder return cmd } -func runAttach(ctx context.Context, opts *attachOptions) error { +func runAttach(ctx context.Context, opts attachOptions) error { ctx, logger := opts.WithContext(ctx) annotations, err := opts.LoadManifestAnnotations() if err != nil { diff --git a/cmd/oras/root/blob/delete.go b/cmd/oras/root/blob/delete.go index c39dbc331..244774a07 100644 --- a/cmd/oras/root/blob/delete.go +++ b/cmd/oras/root/blob/delete.go @@ -62,7 +62,7 @@ Example - Delete a blob and print its descriptor: }, RunE: func(cmd *cobra.Command, args []string) error { opts.targetRef = args[0] - return deleteBlob(cmd.Context(), &opts) + return deleteBlob(cmd.Context(), opts) }, } @@ -70,7 +70,7 @@ Example - Delete a blob and print its descriptor: return cmd } -func deleteBlob(ctx context.Context, opts *deleteBlobOptions) (err error) { +func deleteBlob(ctx context.Context, opts deleteBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common) if err != nil { diff --git a/cmd/oras/root/blob/fetch.go b/cmd/oras/root/blob/fetch.go index 7915811a1..d7f08bed7 100644 --- a/cmd/oras/root/blob/fetch.go +++ b/cmd/oras/root/blob/fetch.go @@ -80,7 +80,7 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' }, Aliases: []string{"get"}, RunE: func(cmd *cobra.Command, args []string) error { - return fetchBlob(cmd.Context(), &opts) + return fetchBlob(cmd.Context(), opts) }, } @@ -89,7 +89,7 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' return cmd } -func fetchBlob(ctx context.Context, opts *fetchBlobOptions) (fetchErr error) { +func fetchBlob(ctx context.Context, opts fetchBlobOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) var target oras.ReadOnlyTarget target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) diff --git a/cmd/oras/root/blob/push.go b/cmd/oras/root/blob/push.go index 2a060aeda..ea3bffaee 100644 --- a/cmd/oras/root/blob/push.go +++ b/cmd/oras/root/blob/push.go @@ -85,7 +85,7 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir': return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return pushBlob(cmd.Context(), &opts) + return pushBlob(cmd.Context(), opts) }, } @@ -95,7 +95,7 @@ Example - Push blob 'hi.txt' into an OCI image layout folder 'layout-dir': return cmd } -func pushBlob(ctx context.Context, opts *pushBlobOptions) (err error) { +func pushBlob(ctx context.Context, opts pushBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewTarget(opts.Common, logger) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 15aa1bac9..ab24c93d4 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -86,7 +86,7 @@ Example - Copy an artifact with multiple tags with concurrency tuned: return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return runCopy(cmd.Context(), &opts) + return runCopy(cmd.Context(), opts) }, } cmd.Flags().BoolVarP(&opts.recursive, "recursive", "r", false, "[Preview] recursively copy the artifact and its referrer artifacts") @@ -96,7 +96,7 @@ Example - Copy an artifact with multiple tags with concurrency tuned: return cmd } -func runCopy(ctx context.Context, opts *copyOptions) error { +func runCopy(ctx context.Context, opts copyOptions) error { ctx, logger := opts.WithContext(ctx) // Prepare source diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index 63676d9d8..1640c4511 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -79,7 +79,7 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return runDiscover(cmd.Context(), &opts) + return runDiscover(cmd.Context(), opts) }, } @@ -90,7 +90,7 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout return cmd } -func runDiscover(ctx context.Context, opts *discoverOptions) error { +func runDiscover(ctx context.Context, opts discoverOptions) error { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { @@ -110,7 +110,7 @@ func runDiscover(ctx context.Context, opts *discoverOptions) error { if opts.outputType == "tree" { root := tree.New(fmt.Sprintf("%s@%s", opts.Path, desc.Digest)) - err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, opts) + err = opts.fetchAllReferrers(ctx, repo, desc, opts.artifactType, root) if err != nil { return err } @@ -138,7 +138,7 @@ func runDiscover(ctx context.Context, opts *discoverOptions) error { return nil } -func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error { +func (opts *discoverOptions) fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node) error { results, err := graph.Referrers(ctx, repo, desc, artifactType) if err != nil { return err @@ -156,14 +156,14 @@ func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc referrerNode.AddPath(strings.TrimSpace(string(bytes))) } } - err := fetchAllReferrers( + err := opts.fetchAllReferrers( ctx, repo, ocispec.Descriptor{ Digest: r.Digest, Size: r.Size, MediaType: r.MediaType, }, - artifactType, referrerNode, opts) + artifactType, referrerNode) if err != nil { return err } diff --git a/cmd/oras/root/login.go b/cmd/oras/root/login.go index 52cca58ca..6cecc2c16 100644 --- a/cmd/oras/root/login.go +++ b/cmd/oras/root/login.go @@ -67,14 +67,14 @@ Example - Log in with username and password in an interactive terminal and no TL }, RunE: func(cmd *cobra.Command, args []string) error { opts.Hostname = args[0] - return runLogin(cmd.Context(), &opts) + return runLogin(cmd.Context(), opts) }, } option.ApplyFlags(&opts, cmd.Flags()) return cmd } -func runLogin(ctx context.Context, opts *loginOptions) (err error) { +func runLogin(ctx context.Context, opts loginOptions) (err error) { ctx, logger := opts.WithContext(ctx) // prompt for credential diff --git a/cmd/oras/root/logout.go b/cmd/oras/root/logout.go index b4d4585ec..bdb39d1f5 100644 --- a/cmd/oras/root/logout.go +++ b/cmd/oras/root/logout.go @@ -44,7 +44,7 @@ Example - Logout: Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { opts.hostname = args[0] - return runLogout(cmd.Context(), &opts) + return runLogout(cmd.Context(), opts) }, } @@ -53,7 +53,7 @@ Example - Logout: return cmd } -func runLogout(ctx context.Context, opts *logoutOptions) error { +func runLogout(ctx context.Context, opts logoutOptions) error { if opts.debug { logrus.SetLevel(logrus.DebugLevel) } diff --git a/cmd/oras/root/manifest/delete.go b/cmd/oras/root/manifest/delete.go index 9fd8b48f2..54ea4dda6 100644 --- a/cmd/oras/root/manifest/delete.go +++ b/cmd/oras/root/manifest/delete.go @@ -66,7 +66,7 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 }, RunE: func(cmd *cobra.Command, args []string) error { opts.targetRef = args[0] - return deleteManifest(cmd.Context(), &opts) + return deleteManifest(cmd.Context(), opts) }, } @@ -75,7 +75,7 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 return cmd } -func deleteManifest(ctx context.Context, opts *deleteOptions) error { +func deleteManifest(ctx context.Context, opts deleteOptions) error { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common) if err != nil { diff --git a/cmd/oras/root/manifest/fetch.go b/cmd/oras/root/manifest/fetch.go index 6d8111add..85a512e78 100644 --- a/cmd/oras/root/manifest/fetch.go +++ b/cmd/oras/root/manifest/fetch.go @@ -79,7 +79,7 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': }, Aliases: []string{"get"}, RunE: func(cmd *cobra.Command, args []string) error { - return fetchManifest(cmd.Context(), &opts) + return fetchManifest(cmd.Context(), opts) }, } @@ -89,7 +89,7 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': return cmd } -func fetchManifest(ctx context.Context, opts *fetchOptions) (fetchErr error) { +func fetchManifest(ctx context.Context, opts fetchOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) diff --git a/cmd/oras/root/manifest/fetch_config.go b/cmd/oras/root/manifest/fetch_config.go index 41a2cddfc..a90e287d3 100644 --- a/cmd/oras/root/manifest/fetch_config.go +++ b/cmd/oras/root/manifest/fetch_config.go @@ -76,7 +76,7 @@ Example - Fetch and print the prettified descriptor of the config: return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return fetchConfig(cmd.Context(), &opts) + return fetchConfig(cmd.Context(), opts) }, } @@ -85,7 +85,7 @@ Example - Fetch and print the prettified descriptor of the config: return cmd } -func fetchConfig(ctx context.Context, opts *fetchConfigOptions) (fetchErr error) { +func fetchConfig(ctx context.Context, opts fetchConfigOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) diff --git a/cmd/oras/root/manifest/push.go b/cmd/oras/root/manifest/push.go index ad1647f6c..d9e737918 100644 --- a/cmd/oras/root/manifest/push.go +++ b/cmd/oras/root/manifest/push.go @@ -92,7 +92,7 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return pushManifest(cmd.Context(), &opts) + return pushManifest(cmd.Context(), opts) }, } @@ -103,7 +103,7 @@ Example - Push a manifest to an OCI image layout folder 'layout-dir' and tag wit return cmd } -func pushManifest(ctx context.Context, opts *pushOptions) error { +func pushManifest(ctx context.Context, opts pushOptions) error { ctx, logger := opts.WithContext(ctx) var target oras.Target var err error diff --git a/cmd/oras/root/pull.go b/cmd/oras/root/pull.go index 7fd317cd4..7fb19dc93 100644 --- a/cmd/oras/root/pull.go +++ b/cmd/oras/root/pull.go @@ -88,7 +88,7 @@ Example - Pull artifact files from an OCI layout archive 'layout.tar': return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return runPull(cmd.Context(), &opts) + return runPull(cmd.Context(), opts) }, } @@ -102,7 +102,7 @@ Example - Pull artifact files from an OCI layout archive 'layout.tar': return cmd } -func runPull(ctx context.Context, opts *pullOptions) error { +func runPull(ctx context.Context, opts pullOptions) error { ctx, logger := opts.WithContext(ctx) // Copy Options var printed sync.Map diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 5c1559333..5bad2955e 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -114,7 +114,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return nil }, RunE: func(cmd *cobra.Command, args []string) error { - return runPush(cmd.Context(), &opts) + return runPush(cmd.Context(), opts) }, } cmd.Flags().StringVarP(&opts.manifestConfigRef, "config", "", "", "`path` of image config file") @@ -125,7 +125,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return cmd } -func runPush(ctx context.Context, opts *pushOptions) error { +func runPush(ctx context.Context, opts pushOptions) error { ctx, logger := opts.WithContext(ctx) annotations, err := opts.LoadManifestAnnotations() if err != nil { diff --git a/cmd/oras/root/repo/ls.go b/cmd/oras/root/repo/ls.go index 32031c722..51539dfb4 100644 --- a/cmd/oras/root/repo/ls.go +++ b/cmd/oras/root/repo/ls.go @@ -59,7 +59,7 @@ Example - List the repositories under the registry that include values lexically if opts.hostname, opts.namespace, err = repository.ParseRepoPath(args[0]); err != nil { return fmt.Errorf("could not parse repository path: %w", err) } - return listRepository(cmd.Context(), &opts) + return listRepository(cmd.Context(), opts) }, } @@ -68,7 +68,7 @@ Example - List the repositories under the registry that include values lexically return cmd } -func listRepository(ctx context.Context, opts *repositoryOptions) error { +func listRepository(ctx context.Context, opts repositoryOptions) error { ctx, logger := opts.WithContext(ctx) reg, err := opts.Remote.NewRegistry(opts.hostname, logger, opts.Common) if err != nil { diff --git a/cmd/oras/root/repo/tags.go b/cmd/oras/root/repo/tags.go index ed98a757b..797f02adf 100644 --- a/cmd/oras/root/repo/tags.go +++ b/cmd/oras/root/repo/tags.go @@ -68,7 +68,7 @@ Example - [Experimental] Show tags associated with a digest: return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return showTags(cmd.Context(), &opts) + return showTags(cmd.Context(), opts) }, } cmd.Flags().StringVar(&opts.last, "last", "", "start after the tag specified by `last`") @@ -77,7 +77,7 @@ Example - [Experimental] Show tags associated with a digest: return cmd } -func showTags(ctx context.Context, opts *showTagsOptions) error { +func showTags(ctx context.Context, opts showTagsOptions) error { ctx, logger := opts.WithContext(ctx) finder, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) if err != nil { diff --git a/cmd/oras/root/tag.go b/cmd/oras/root/tag.go index 28ec967d4..4524d4fb9 100644 --- a/cmd/oras/root/tag.go +++ b/cmd/oras/root/tag.go @@ -66,7 +66,7 @@ Example - Tag the manifest 'v1.0.1' to 'v1.0.2' in an OCI image layout folder 'l return option.Parse(&opts) }, RunE: func(cmd *cobra.Command, args []string) error { - return tagManifest(cmd.Context(), &opts) + return tagManifest(cmd.Context(), opts) }, } @@ -75,7 +75,7 @@ Example - Tag the manifest 'v1.0.1' to 'v1.0.2' in an OCI image layout folder 'l return cmd } -func tagManifest(ctx context.Context, opts *tagOptions) error { +func tagManifest(ctx context.Context, opts tagOptions) error { ctx, logger := opts.WithContext(ctx) target, err := opts.NewTarget(opts.Common, logger) if err != nil { From e8a3a3007b39cf5faf0535039de264093f31f466 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 22 Aug 2023 15:14:09 +0000 Subject: [PATCH 39/44] deduplicate warning while copying Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 2 +- cmd/oras/internal/option/target.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 927780f9d..cb4e30f6c 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -67,6 +67,7 @@ func (opts *Remote) EnableDistributionSpecFlag() { func (opts *Remote) ApplyFlags(fs *pflag.FlagSet) { opts.ApplyFlagsWithPrefix(fs, "", "") fs.BoolVarP(&opts.PasswordFromStdin, "password-stdin", "", false, "read password or identity token from stdin") + opts.warned = &sync.Map{} } func applyPrefix(prefix, description string) (flagPrefix, notePrefix string) { @@ -107,7 +108,6 @@ func (opts *Remote) ApplyFlagsWithPrefix(fs *pflag.FlagSet, prefix, description // Parse tries to read password with optional cmd prompt. func (opts *Remote) Parse() error { - opts.warned = &sync.Map{} if err := opts.parseCustomHeaders(); err != nil { return err } diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index df5e676ae..bebdb547d 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "strings" + "sync" "github.com/sirupsen/logrus" "github.com/spf13/pflag" @@ -200,6 +201,8 @@ func (opts *BinaryTarget) ApplyFlags(fs *pflag.FlagSet) { opts.From.ApplyFlagsWithPrefix(fs, "from", "source") opts.To.ApplyFlagsWithPrefix(fs, "to", "destination") fs.StringArrayVarP(&opts.resolveFlag, "resolve", "", nil, "base DNS rules formatted in `host:port:address[:address_port]` for --from-resolve and --to-resolve") + opts.From.warned = &sync.Map{} + opts.To.warned = opts.From.warned } // Parse parses user-provided flags and arguments into option struct. From 1ff46ffdae425bf29253b2f4697d314fe4dd84ba Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 23 Aug 2023 16:22:25 +0800 Subject: [PATCH 40/44] reorder parameters Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 4 ++-- cmd/oras/internal/option/remote_test.go | 6 +++--- cmd/oras/internal/option/target.go | 6 +++--- cmd/oras/root/blob/delete.go | 2 +- cmd/oras/root/blob/fetch.go | 2 +- cmd/oras/root/cp.go | 2 +- cmd/oras/root/discover.go | 2 +- cmd/oras/root/login.go | 2 +- cmd/oras/root/manifest/delete.go | 2 +- cmd/oras/root/manifest/fetch.go | 2 +- cmd/oras/root/manifest/fetch_config.go | 2 +- cmd/oras/root/pull.go | 2 +- cmd/oras/root/repo/ls.go | 2 +- cmd/oras/root/repo/tags.go | 2 +- 14 files changed, 19 insertions(+), 19 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index cb4e30f6c..76dd90501 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -263,7 +263,7 @@ func (opts *Remote) handleWarning(registry string, logger logrus.FieldLogger) fu } // NewRegistry assembles a oras remote registry. -func (opts *Remote) NewRegistry(registry string, logger logrus.FieldLogger, common Common) (reg *remote.Registry, err error) { +func (opts *Remote) NewRegistry(registry string, common Common, logger logrus.FieldLogger) (reg *remote.Registry, err error) { reg, err = remote.NewRegistry(registry) if err != nil { return nil, err @@ -278,7 +278,7 @@ func (opts *Remote) NewRegistry(registry string, logger logrus.FieldLogger, comm } // NewRepository assembles a oras remote repository. -func (opts *Remote) NewRepository(reference string, logger logrus.FieldLogger, common Common) (repo *remote.Repository, err error) { +func (opts *Remote) NewRepository(reference string, common Common, logger logrus.FieldLogger) (repo *remote.Repository, err error) { repo, err = remote.NewRepository(reference) if err != nil { return nil, err diff --git a/cmd/oras/internal/option/remote_test.go b/cmd/oras/internal/option/remote_test.go index 63881a0ba..9c60d939b 100644 --- a/cmd/oras/internal/option/remote_test.go +++ b/cmd/oras/internal/option/remote_test.go @@ -181,7 +181,7 @@ func TestRemote_NewRegistry(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - reg, err := opts.NewRegistry(uri.Host, logrus.New(), opts.Common) + reg, err := opts.NewRegistry(uri.Host, opts.Common, logrus.New()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -209,7 +209,7 @@ func TestRemote_NewRepository(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - repo, err := opts.NewRepository(uri.Host+"/"+testRepo, logrus.New(), opts.Common) + repo, err := opts.NewRepository(uri.Host+"/"+testRepo, opts.Common, logrus.New()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -256,7 +256,7 @@ func TestRemote_NewRepository_Retry(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - repo, err := opts.NewRepository(uri.Host+"/"+testRepo, logrus.New(), opts.Common) + repo, err := opts.NewRepository(uri.Host+"/"+testRepo, opts.Common, logrus.New()) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index bebdb547d..a3c83c635 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -123,7 +123,7 @@ func (opts *Target) NewTarget(common Common, logger logrus.FieldLogger) (oras.Gr } return oci.New(opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, logger, common) + repo, err := opts.NewRepository(opts.RawReference, common, logger) if err != nil { return nil, err } @@ -144,7 +144,7 @@ type ReadOnlyGraphTagFinderTarget interface { } // NewReadonlyTargets generates a new read only target based on opts. -func (opts *Target) NewReadonlyTarget(ctx context.Context, logger logrus.FieldLogger, common Common) (ReadOnlyGraphTagFinderTarget, error) { +func (opts *Target) NewReadonlyTarget(ctx context.Context, common Common, logger logrus.FieldLogger) (ReadOnlyGraphTagFinderTarget, error) { switch opts.Type { case TargetTypeOCILayout: var err error @@ -161,7 +161,7 @@ func (opts *Target) NewReadonlyTarget(ctx context.Context, logger logrus.FieldLo } return oci.NewFromTar(ctx, opts.Path) case TargetTypeRemote: - repo, err := opts.NewRepository(opts.RawReference, logger, common) + repo, err := opts.NewRepository(opts.RawReference, common, logger) if err != nil { return nil, err } diff --git a/cmd/oras/root/blob/delete.go b/cmd/oras/root/blob/delete.go index 244774a07..e1fbb09ac 100644 --- a/cmd/oras/root/blob/delete.go +++ b/cmd/oras/root/blob/delete.go @@ -72,7 +72,7 @@ Example - Delete a blob and print its descriptor: func deleteBlob(ctx context.Context, opts deleteBlobOptions) (err error) { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common) + repo, err := opts.NewRepository(opts.targetRef, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/blob/fetch.go b/cmd/oras/root/blob/fetch.go index d7f08bed7..d43ae382d 100644 --- a/cmd/oras/root/blob/fetch.go +++ b/cmd/oras/root/blob/fetch.go @@ -92,7 +92,7 @@ Example - Fetch and print a blob from OCI image layout archive file 'layout.tar' func fetchBlob(ctx context.Context, opts fetchBlobOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) var target oras.ReadOnlyTarget - target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index ab24c93d4..90f7cb090 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -100,7 +100,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { ctx, logger := opts.WithContext(ctx) // Prepare source - src, err := opts.From.NewReadonlyTarget(ctx, logger, opts.Common) + src, err := opts.From.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index 1640c4511..5ebdd35b0 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -92,7 +92,7 @@ Example - Discover referrers of the manifest tagged 'v1' in an OCI image layout func runDiscover(ctx context.Context, opts discoverOptions) error { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) + repo, err := opts.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/login.go b/cmd/oras/root/login.go index 6cecc2c16..d6cc65d5d 100644 --- a/cmd/oras/root/login.go +++ b/cmd/oras/root/login.go @@ -108,7 +108,7 @@ func runLogin(ctx context.Context, opts loginOptions) (err error) { if err != nil { return err } - remote, err := opts.Remote.NewRegistry(opts.Hostname, logger, opts.Common) + remote, err := opts.Remote.NewRegistry(opts.Hostname, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/manifest/delete.go b/cmd/oras/root/manifest/delete.go index 54ea4dda6..4458fae77 100644 --- a/cmd/oras/root/manifest/delete.go +++ b/cmd/oras/root/manifest/delete.go @@ -77,7 +77,7 @@ Example - Delete a manifest by digest 'sha256:99e4703fbf30916f549cd6bfa9cdbab614 func deleteManifest(ctx context.Context, opts deleteOptions) error { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewRepository(opts.targetRef, logger, opts.Common) + repo, err := opts.NewRepository(opts.targetRef, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/manifest/fetch.go b/cmd/oras/root/manifest/fetch.go index 85a512e78..52ec2f80b 100644 --- a/cmd/oras/root/manifest/fetch.go +++ b/cmd/oras/root/manifest/fetch.go @@ -92,7 +92,7 @@ Example - Fetch raw manifest from an OCI layout archive file 'layout.tar': func fetchManifest(ctx context.Context, opts fetchOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) - target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/manifest/fetch_config.go b/cmd/oras/root/manifest/fetch_config.go index a90e287d3..e0f871f34 100644 --- a/cmd/oras/root/manifest/fetch_config.go +++ b/cmd/oras/root/manifest/fetch_config.go @@ -88,7 +88,7 @@ Example - Fetch and print the prettified descriptor of the config: func fetchConfig(ctx context.Context, opts fetchConfigOptions) (fetchErr error) { ctx, logger := opts.WithContext(ctx) - repo, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) + repo, err := opts.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/pull.go b/cmd/oras/root/pull.go index 7fb19dc93..6baf93091 100644 --- a/cmd/oras/root/pull.go +++ b/cmd/oras/root/pull.go @@ -182,7 +182,7 @@ func runPull(ctx context.Context, opts pullOptions) error { return ret, nil } - target, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) + target, err := opts.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/repo/ls.go b/cmd/oras/root/repo/ls.go index 51539dfb4..cea8dd334 100644 --- a/cmd/oras/root/repo/ls.go +++ b/cmd/oras/root/repo/ls.go @@ -70,7 +70,7 @@ Example - List the repositories under the registry that include values lexically func listRepository(ctx context.Context, opts repositoryOptions) error { ctx, logger := opts.WithContext(ctx) - reg, err := opts.Remote.NewRegistry(opts.hostname, logger, opts.Common) + reg, err := opts.Remote.NewRegistry(opts.hostname, opts.Common, logger) if err != nil { return err } diff --git a/cmd/oras/root/repo/tags.go b/cmd/oras/root/repo/tags.go index 797f02adf..45e37dba9 100644 --- a/cmd/oras/root/repo/tags.go +++ b/cmd/oras/root/repo/tags.go @@ -79,7 +79,7 @@ Example - [Experimental] Show tags associated with a digest: func showTags(ctx context.Context, opts showTagsOptions) error { ctx, logger := opts.WithContext(ctx) - finder, err := opts.NewReadonlyTarget(ctx, logger, opts.Common) + finder, err := opts.NewReadonlyTarget(ctx, opts.Common, logger) if err != nil { return err } From 9a9b387b305bbae014f2962d6116a14ab63fe5de Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 23 Aug 2023 16:27:59 +0800 Subject: [PATCH 41/44] lazy initialized dedup map Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 17 +++++++++++------ cmd/oras/internal/option/target.go | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index 76dd90501..d5c246204 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -55,7 +55,7 @@ type Remote struct { distributionSpec distributionSpec headerFlags []string headers http.Header - warned *sync.Map + warned map[string]*sync.Map } // EnableDistributionSpecFlag set distribution specification flag as applicable. @@ -67,7 +67,6 @@ func (opts *Remote) EnableDistributionSpecFlag() { func (opts *Remote) ApplyFlags(fs *pflag.FlagSet) { opts.ApplyFlagsWithPrefix(fs, "", "") fs.BoolVarP(&opts.PasswordFromStdin, "password-stdin", "", false, "read password or identity token from stdin") - opts.warned = &sync.Map{} } func applyPrefix(prefix, description string) (flagPrefix, notePrefix string) { @@ -250,13 +249,19 @@ func (opts *Remote) parseCustomHeaders() error { func (opts *Remote) Credential() auth.Credential { return credential.Credential(opts.Username, opts.Password) } + func (opts *Remote) handleWarning(registry string, logger logrus.FieldLogger) func(warning remote.Warning) { + if opts.warned == nil { + opts.warned = make(map[string]*sync.Map) + } + warned := opts.warned[registry] + if warned == nil { + warned = &sync.Map{} + opts.warned[registry] = warned + } logger = logger.WithField("registry", registry) return func(warning remote.Warning) { - if _, loaded := opts.warned.LoadOrStore(struct { - string - remote.WarningValue - }{registry, warning.WarningValue}, true); !loaded { + if _, loaded := warned.LoadOrStore(warning.WarningValue, struct{}{}); !loaded { logger.Warn(warning.Text) } } diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index a3c83c635..01a5c83c6 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -201,7 +201,7 @@ func (opts *BinaryTarget) ApplyFlags(fs *pflag.FlagSet) { opts.From.ApplyFlagsWithPrefix(fs, "from", "source") opts.To.ApplyFlagsWithPrefix(fs, "to", "destination") fs.StringArrayVarP(&opts.resolveFlag, "resolve", "", nil, "base DNS rules formatted in `host:port:address[:address_port]` for --from-resolve and --to-resolve") - opts.From.warned = &sync.Map{} + opts.From.warned = make(map[string]*sync.Map) opts.To.warned = opts.From.warned } From 8eedab49e1394dfcc0ea6b2053a7383118249279 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 23 Aug 2023 16:53:07 +0800 Subject: [PATCH 42/44] update target parsing Signed-off-by: Billy Zha --- cmd/oras/internal/option/target.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/oras/internal/option/target.go b/cmd/oras/internal/option/target.go index 01a5c83c6..36290204e 100644 --- a/cmd/oras/internal/option/target.go +++ b/cmd/oras/internal/option/target.go @@ -201,12 +201,12 @@ func (opts *BinaryTarget) ApplyFlags(fs *pflag.FlagSet) { opts.From.ApplyFlagsWithPrefix(fs, "from", "source") opts.To.ApplyFlagsWithPrefix(fs, "to", "destination") fs.StringArrayVarP(&opts.resolveFlag, "resolve", "", nil, "base DNS rules formatted in `host:port:address[:address_port]` for --from-resolve and --to-resolve") - opts.From.warned = make(map[string]*sync.Map) - opts.To.warned = opts.From.warned } // Parse parses user-provided flags and arguments into option struct. func (opts *BinaryTarget) Parse() error { + opts.From.warned = make(map[string]*sync.Map) + opts.To.warned = opts.From.warned // resolve are parsed in array order, latter will overwrite former opts.From.resolveFlag = append(opts.resolveFlag, opts.From.resolveFlag...) opts.To.resolveFlag = append(opts.resolveFlag, opts.To.resolveFlag...) From 9f62b98748042c04e13da35ff1bb8fda3c1e6dbf Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 23 Aug 2023 16:57:57 +0800 Subject: [PATCH 43/44] revert Signed-off-by: Billy Zha --- cmd/oras/root/discover.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/oras/root/discover.go b/cmd/oras/root/discover.go index 5ebdd35b0..801710ead 100644 --- a/cmd/oras/root/discover.go +++ b/cmd/oras/root/discover.go @@ -110,7 +110,7 @@ func runDiscover(ctx context.Context, opts discoverOptions) error { if opts.outputType == "tree" { root := tree.New(fmt.Sprintf("%s@%s", opts.Path, desc.Digest)) - err = opts.fetchAllReferrers(ctx, repo, desc, opts.artifactType, root) + err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, &opts) if err != nil { return err } @@ -138,7 +138,7 @@ func runDiscover(ctx context.Context, opts discoverOptions) error { return nil } -func (opts *discoverOptions) fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node) error { +func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error { results, err := graph.Referrers(ctx, repo, desc, artifactType) if err != nil { return err @@ -156,14 +156,14 @@ func (opts *discoverOptions) fetchAllReferrers(ctx context.Context, repo oras.Re referrerNode.AddPath(strings.TrimSpace(string(bytes))) } } - err := opts.fetchAllReferrers( + err := fetchAllReferrers( ctx, repo, ocispec.Descriptor{ Digest: r.Digest, Size: r.Size, MediaType: r.MediaType, }, - artifactType, referrerNode) + artifactType, referrerNode, opts) if err != nil { return err } From be424277b6040818336bd70675b7e567faea862b Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 23 Aug 2023 16:59:35 +0800 Subject: [PATCH 44/44] code clean Signed-off-by: Billy Zha --- cmd/oras/internal/option/remote.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/oras/internal/option/remote.go b/cmd/oras/internal/option/remote.go index d5c246204..f56b90bf5 100644 --- a/cmd/oras/internal/option/remote.go +++ b/cmd/oras/internal/option/remote.go @@ -290,11 +290,11 @@ func (opts *Remote) NewRepository(reference string, common Common, logger logrus } registry := repo.Reference.Registry repo.PlainHTTP = opts.isPlainHttp(registry) - repo.SkipReferrersGC = true + repo.HandleWarning = opts.handleWarning(registry, logger) if repo.Client, err = opts.authClient(registry, common.Debug); err != nil { return nil, err } - repo.HandleWarning = opts.handleWarning(registry, logger) + repo.SkipReferrersGC = true if opts.distributionSpec.referrersAPI != nil { if err := repo.SetReferrersCapability(*opts.distributionSpec.referrersAPI); err != nil { return nil, err