From eb25c8d77efe5784e64f56909c0d4dc2860c16dc Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 03:32:12 +0000 Subject: [PATCH 01/23] feat: pack OCI image spec v1.1 manifests Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 26 +++++++---------- cmd/oras/root/attach.go | 3 +- cmd/oras/root/push.go | 22 +++++++------- go.mod | 4 +-- go.sum | 8 ++--- internal/graph/graph.go | 28 +----------------- internal/graph/graph_test.go | 50 ++++---------------------------- 7 files changed, 35 insertions(+), 106 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index ccbb204fb..f403f9813 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -18,35 +18,31 @@ package option import ( "fmt" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/pflag" ) +const ( + V1_1 = "1.1" + V1_0 = "1.0" +) + // ImageSpec option struct. type ImageSpec struct { - // Manifest type for building artifact - ManifestMediaType string - - // specFlag should be provided in form of `-` - specFlag string + Flag string } // Parse parses flags into the option. func (opts *ImageSpec) Parse() error { - switch opts.specFlag { - case "v1.1-image": - opts.ManifestMediaType = ocispec.MediaTypeImageManifest - case "v1.1-artifact": - opts.ManifestMediaType = ocispec.MediaTypeArtifactManifest - default: - return fmt.Errorf("unknown image specification flag: %q", opts.specFlag) + switch opts.Flag { + case V1_1, V1_0: + return nil } - return nil + return fmt.Errorf("unknown image specification flag: %q", opts.Flag) } // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.specFlag, "image-spec", "v1.1-image", "[Experimental] specify manifest type for building artifact. options: v1.1-image, v1.1-artifact") + fs.StringVar(&opts.Flag, "image-spec", V1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", V1_1, V1_0)) } // distributionSpec option struct. diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index d0a84dd13..a18be3e0d 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -35,7 +35,6 @@ import ( type attachOptions struct { option.Common option.Packer - option.ImageSpec option.Target option.Referrers @@ -139,7 +138,7 @@ func runAttach(ctx context.Context, opts attachOptions) error { packOpts := oras.PackOptions{ Subject: &subject, ManifestAnnotations: annotations[option.AnnotationManifest], - PackImageManifest: opts.ManifestMediaType == ocispec.MediaTypeImageManifest, + PackImageManifest: true, } pack := func() (ocispec.Descriptor, error) { return oras.Pack(ctx, store, opts.artifactType, descs, packOpts) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 0423df717..b0d95bfd4 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -102,15 +102,16 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t opts.RawReference = refs[0] opts.extraRefs = refs[1:] opts.FileRefs = args[1:] - if opts.manifestConfigRef != "" { - if opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided") - } - if opts.ManifestMediaType == ocispec.MediaTypeArtifactManifest { - return errors.New("cannot build an OCI artifact with manifest config") - } + + if err := option.Parse(&opts); err != nil { + return err + } + + if opts.ImageSpec.Flag == option.V1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") } - return option.Parse(&opts) + return nil + }, RunE: func(cmd *cobra.Command, args []string) error { return runPush(cmd.Context(), opts) @@ -135,6 +136,7 @@ func runPush(ctx context.Context, opts pushOptions) error { packOpts := oras.PackOptions{ ConfigAnnotations: annotations[option.AnnotationConfig], ManifestAnnotations: annotations[option.AnnotationManifest], + PackImageManifest: true, } store, err := file.New("") if err != nil { @@ -152,10 +154,6 @@ func runPush(ctx context.Context, opts pushOptions) error { } desc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &desc - packOpts.PackImageManifest = true - } - if opts.ManifestMediaType == ocispec.MediaTypeImageManifest { - packOpts.PackImageManifest = true } descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { diff --git a/go.mod b/go.mod index d76346c6e..1f32276f6 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.20 require ( github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0-rc4 github.com/oras-project/oras-credentials-go v0.2.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 golang.org/x/term v0.10.0 gopkg.in/yaml.v3 v3.0.1 - oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b + oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 ) require ( diff --git a/go.sum b/go.sum index ef1f32966..66806f226 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/oras-project/oras-credentials-go v0.2.0 h1:BvWAXo0e5unWR6Hfxyb0K04mHNHreQz/Zclw6IzCYJo= github.com/oras-project/oras-credentials-go v0.2.0/go.mod h1:JVdg7a5k7hzTrEeeouwag0aCv7OLrS77r7/6w3gVirU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -34,5 +34,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b h1:NYynybpqtG3lLTZMWNlrvUlcyGakCke57tg4TX6w2kA= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b/go.mod h1:goptA58HogB/6sLN7KV6FgoiurcVxd5QBqv8wMVB0as= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 h1:lK/VtXN9+dzSD3EMuCXu0D28OJadBxB9yUpaJBDIABM= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ= diff --git a/internal/graph/graph.go b/internal/graph/graph.go index bcefdd43d..731df5faf 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -43,18 +43,6 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr nodes = manifest.Layers subject = manifest.Subject config = &manifest.Config - case ocispec.MediaTypeArtifactManifest: - var fetched []byte - fetched, err = content.FetchAll(ctx, fetcher, node) - if err != nil { - return - } - var manifest ocispec.Artifact - if err = json.Unmarshal(fetched, &manifest); err != nil { - return - } - nodes = manifest.Blobs - subject = manifest.Subject default: nodes, err = content.Successors(ctx, fetcher, node) } @@ -83,20 +71,6 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc } for _, node := range predecessors { switch node.MediaType { - case ocispec.MediaTypeArtifactManifest: - fetched, err := fetchBytes(ctx, target, node) - if err != nil { - return nil, err - } - var artifact ocispec.Artifact - if err := json.Unmarshal(fetched, &artifact); err != nil { - return nil, err - } - if artifact.Subject == nil || !content.Equal(*artifact.Subject, desc) { - continue - } - node.ArtifactType = artifact.ArtifactType - node.Annotations = artifact.Annotations case ocispec.MediaTypeImageManifest: fetched, err := fetchBytes(ctx, target, node) if err != nil { @@ -141,7 +115,7 @@ func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStor } for _, node := range predecessors { switch node.MediaType { - case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: + case ocispec.MediaTypeImageManifest: results = append(results, node) } } diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 599307fa2..1c7e14ab3 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -89,19 +89,6 @@ func TestReferrers(t *testing.T) { } appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) } - generateArtifact := func(artifactType string, subject *ocispec.Descriptor, annotations map[string]string, blobs ...ocispec.Descriptor) { - manifest := ocispec.Artifact{ - Subject: subject, - Blobs: blobs, - Annotations: annotations, - ArtifactType: artifactType, - } - manifestJSON, err := json.Marshal(manifest) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON) - } generateIndex := func(manifests ...ocispec.Descriptor) { index := ocispec.Index{ Manifests: manifests, @@ -117,11 +104,10 @@ func TestReferrers(t *testing.T) { imgConfig subject image - artifact index ) anno := map[string]string{"test": "foo"} - appendBlob(ocispec.MediaTypeArtifactManifest, []byte(`{"name":"subject content"}`)) + appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) imageType := "test.image" appendBlob(imageType, []byte("config content")) generateImage(nil, nil, descs[imgConfig], descs[blob]) @@ -129,12 +115,7 @@ func TestReferrers(t *testing.T) { imageDesc := descs[image] imageDesc.Annotations = anno imageDesc.ArtifactType = imageType - artifactType := "test.artifact" - generateArtifact(artifactType, &descs[subject], anno, descs[blob]) generateIndex(descs[subject]) - artifactDesc := descs[artifact] - artifactDesc.Annotations = anno - artifactDesc.ArtifactType = artifactType referrers := []ocispec.Descriptor{descs[image], descs[image]} memory := memory.New() @@ -163,7 +144,6 @@ func TestReferrers(t *testing.T) { {"should return nil for config node", args{ctx, finder, descs[imgConfig], ""}, nil, false}, {"should return nil for blob/layer node", args{ctx, finder, descs[blob], ""}, nil, false}, {"should find filtered image referrer", args{ctx, finder, descs[subject], imageType}, []ocispec.Descriptor{imageDesc}, false}, - {"should find filtered artifact referrer", args{ctx, finder, descs[subject], artifactType}, []ocispec.Descriptor{artifactDesc}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -179,15 +159,14 @@ func TestReferrers(t *testing.T) { } t.Run("should find referrers in predecessors", func(t *testing.T) { - want1 := []ocispec.Descriptor{artifactDesc, imageDesc} - want2 := []ocispec.Descriptor{imageDesc, artifactDesc} + want := []ocispec.Descriptor{imageDesc} got, err := Referrers(ctx, finder, descs[subject], "") if err != nil { t.Errorf("Referrers() error = %v", err) return } - if !reflect.DeepEqual(got, want1) && !reflect.DeepEqual(got, want2) { - t.Errorf("Referrers() = %v, want %v", got, want1) + if !reflect.DeepEqual(got, want) && !reflect.DeepEqual(got, want) { + t.Errorf("Referrers() = %v, want %v", got, want) } }) } @@ -216,18 +195,6 @@ func TestSuccessors(t *testing.T) { } appendBlob(mediaType, manifestJSON) } - generateArtifact := func(artifactType string, subject *ocispec.Descriptor, blobs ...ocispec.Descriptor) { - manifest := ocispec.Artifact{ - MediaType: ocispec.MediaTypeArtifactManifest, - Subject: subject, - Blobs: blobs, - } - manifestJSON, err := json.Marshal(manifest) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON) - } generateIndex := func(manifests ...ocispec.Descriptor) { index := ocispec.Index{ Manifests: manifests, @@ -243,16 +210,13 @@ func TestSuccessors(t *testing.T) { config ociImage dockerImage - artifact index ) - appendBlob(ocispec.MediaTypeArtifactManifest, []byte(`{"name":"subject content"}`)) + appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) imageType := "test.image" appendBlob(imageType, []byte("config content")) generateImage(&descs[subject], ocispec.MediaTypeImageManifest, descs[config]) generateImage(&descs[subject], docker.MediaTypeManifest, descs[config]) - artifactType := "test.artifact" - generateArtifact(artifactType, &descs[subject]) generateIndex(descs[subject]) memory := memory.New() ctx := context.Background() @@ -276,12 +240,10 @@ func TestSuccessors(t *testing.T) { wantConfig *ocispec.Descriptor wantErr bool }{ - {"should failed to get non-existent artifact", args{ctx, fetcher, ocispec.Descriptor{MediaType: ocispec.MediaTypeArtifactManifest}}, nil, nil, nil, true}, {"should failed to get non-existent OCI image", args{ctx, fetcher, ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest}}, nil, nil, nil, true}, {"should failed to get non-existent docker image", args{ctx, fetcher, ocispec.Descriptor{MediaType: docker.MediaTypeManifest}}, nil, nil, nil, true}, {"should get success of a docker image", args{ctx, fetcher, descs[dockerImage]}, nil, &descs[subject], &descs[config], false}, {"should get success of an OCI image", args{ctx, fetcher, descs[ociImage]}, nil, &descs[subject], &descs[config], false}, - {"should get success of an artifact", args{ctx, fetcher, descs[artifact]}, nil, &descs[subject], nil, false}, {"should get success of an index", args{ctx, fetcher, descs[index]}, []ocispec.Descriptor{descs[subject]}, nil, nil, false}, } for _, tt := range tests { @@ -345,7 +307,7 @@ func TestFindReferrerPredecessors(t *testing.T) { image ) var anno map[string]string - appendBlob(ocispec.MediaTypeArtifactManifest, []byte(`{"name":"subject content"}`)) + appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) imageType := "test.image" appendBlob(imageType, []byte("config content")) generateImage(&descs[subject], anno, descs[imgConfig]) From f8be004f5bf8c2dde9ab4dd88bc5fe27cb461730 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 03:38:49 +0000 Subject: [PATCH 02/23] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/attach.go | 44 +++----------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 883ab2fed..6e714c024 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -111,9 +111,7 @@ var _ = Describe("Common registry users:", func() { subjectRef := RegistryRef(Host, testRepo, foobar.Tag) prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). - WithWorkDir(tempDir). - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() + MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() // validate var index ocispec.Index @@ -148,24 +146,6 @@ var _ = Describe("Common registry users:", func() { ExpectFailure(). Exec() }) - - It("should attach a file via a OCI Artifact", func() { - testRepo := attachTestRepo("artifact") - tempDir := PrepareTempFiles() - subjectRef := RegistryRef(Host, testRepo, foobar.Tag) - prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef) - // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-artifact"). - WithWorkDir(tempDir). - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() - - // validate - var index ocispec.Index - bytes := ORAS("discover", subjectRef, "-o", "json").Exec().Out.Contents() - Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) - Expect(len(index.Manifests)).To(Equal(1)) - Expect(index.Manifests[0].MediaType).To(Equal("application/vnd.oci.artifact.manifest.v1+json")) - }) }) }) @@ -177,7 +157,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() @@ -260,7 +240,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image", "--distribution-spec", "v1.1-referrers-tag"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--distribution-spec", "v1.1-referrers-tag"). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() @@ -323,7 +303,7 @@ var _ = Describe("OCI image layout users:", func() { subjectRef := LayoutRef(root, foobar.Tag) prepare(root) // test - ORAS("attach", "--artifact-type", "test.attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", Flags.Layout, subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(root). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() @@ -334,21 +314,5 @@ var _ = Describe("OCI image layout users:", func() { Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].MediaType).To(Equal("application/vnd.oci.image.manifest.v1+json")) }) - It("should attach a file via a OCI Artifact", func() { - root := PrepareTempFiles() - subjectRef := LayoutRef(root, foobar.Tag) - prepare(root) - // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, Flags.Layout, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-artifact"). - WithWorkDir(root). - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() - - // validate - var index ocispec.Index - bytes := ORAS("discover", subjectRef, Flags.Layout, "-o", "json").Exec().Out.Contents() - Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) - Expect(len(index.Manifests)).To(Equal(1)) - Expect(index.Manifests[0].MediaType).To(Equal("application/vnd.oci.artifact.manifest.v1+json")) - }) }) }) From 125f1852fc10b56c3180f053c82f8a3770845dd8 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 03:44:45 +0000 Subject: [PATCH 03/23] fix line Signed-off-by: Billy Zha --- internal/graph/graph_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 1c7e14ab3..370d34bb2 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -165,7 +165,7 @@ func TestReferrers(t *testing.T) { t.Errorf("Referrers() error = %v", err) return } - if !reflect.DeepEqual(got, want) && !reflect.DeepEqual(got, want) { + if !reflect.DeepEqual(got, want) { t.Errorf("Referrers() = %v, want %v", got, want) } }) From e65ee375863c526067572bf338f2fd3c86de6d6f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 04:36:35 +0000 Subject: [PATCH 04/23] fix e2e Signed-off-by: Billy Zha --- internal/graph/graph.go | 69 +++++++++++++++++++++++++++++- test/e2e/suite/command/attach.go | 11 ++--- test/e2e/suite/command/discover.go | 2 +- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 731df5faf..77c054a59 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -25,6 +25,34 @@ import ( "oras.land/oras/internal/docker" ) +// MediaTypeArtifactManifest specifies the media type for a content descriptor. +const MediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json" + +// Artifact describes an artifact manifest. +// This structure provides `application/vnd.oci.artifact.manifest.v1+json` mediatype when marshalled to JSON. +// +// This manifest type was introduced in image-spec v1.1.0-rc1 and was removed in +// image-spec v1.1.0-rc3. It is not part of the current image-spec and is kept +// here for Go compatibility. +// +// Reference: https://github.com/opencontainers/image-spec/pull/999 +type Artifact struct { + // MediaType is the media type of the object this schema refers to. + MediaType string `json:"mediaType"` + + // ArtifactType is the IANA media type of the artifact this schema refers to. + ArtifactType string `json:"artifactType"` + + // Blobs is a collection of blobs referenced by this manifest. + Blobs []ocispec.Descriptor `json:"blobs,omitempty"` + + // Subject (reference) is an optional link from the artifact to another manifest forming an association between the artifact and the other manifest. + Subject *ocispec.Descriptor `json:"subject,omitempty"` + + // Annotations contains arbitrary metadata for the artifact manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + // Successors returns the nodes directly pointed by the current node, picking // out subject and config descriptor if applicable. // Returning nil when no subject and config found. @@ -43,6 +71,18 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr nodes = manifest.Layers subject = manifest.Subject config = &manifest.Config + case MediaTypeArtifactManifest: + var fetched []byte + fetched, err = content.FetchAll(ctx, fetcher, node) + if err != nil { + return + } + var manifest Artifact + if err = json.Unmarshal(fetched, &manifest); err != nil { + return + } + nodes = manifest.Blobs + subject = manifest.Subject default: nodes, err = content.Successors(ctx, fetcher, node) } @@ -71,6 +111,20 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc } for _, node := range predecessors { switch node.MediaType { + case MediaTypeArtifactManifest: + fetched, err := fetchBytes(ctx, target, node) + if err != nil { + return nil, err + } + var artifact Artifact + if err := json.Unmarshal(fetched, &artifact); err != nil { + return nil, err + } + if artifact.Subject == nil || !content.Equal(*artifact.Subject, desc) { + continue + } + node.ArtifactType = artifact.ArtifactType + node.Annotations = artifact.Annotations case ocispec.MediaTypeImageManifest: fetched, err := fetchBytes(ctx, target, node) if err != nil { @@ -115,8 +169,21 @@ func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStor } for _, node := range predecessors { switch node.MediaType { - case ocispec.MediaTypeImageManifest: + case MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: results = append(results, node) + case ocispec.MediaTypeImageIndex: + fetched, err := fetchBytes(ctx, src, node) + if err != nil { + return nil, err + } + // convert to json + var index ocispec.Index + if err := json.Unmarshal(fetched, &index); err != nil { + return nil, err + } + if index.Subject != nil && content.Equal(*index.Subject, desc) { + results = append(results, node) + } } } return results, nil diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 6e714c024..18f2208a6 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -43,7 +43,6 @@ var _ = Describe("ORAS beginners:", func() { It("should show preview and help doc", func() { out := ORAS("attach", "--help").MatchKeyWords(feature.Preview.Mark+" Attach", feature.Preview.Description, ExampleDesc).Exec() gomega.Expect(out).Should(gbytes.Say("--distribution-spec string\\s+%s", regexp.QuoteMeta(feature.Preview.Mark))) - gomega.Expect(out).Should(gbytes.Say("--image-spec string\\s+%s", regexp.QuoteMeta(feature.Experimental.Mark))) }) It("should fail when no subject reference provided", func() { @@ -65,11 +64,6 @@ var _ = Describe("ORAS beginners:", func() { ORAS("attach", "--artifact-type", "oras.test", RegistryRef(Host, ImageRepo, foobar.Tag), "--distribution-spec", "???"). ExpectFailure().MatchErrKeyWords("unknown distribution specification flag").Exec() }) - - It("should fail if image spec is unkown", func() { - ORAS("attach", "--artifact-type", "oras.test", RegistryRef(Host, ImageRepo, foobar.Tag), "--image-spec", "???"). - ExpectFailure().MatchErrKeyWords("unknown image specification flag").Exec() - }) }) }) @@ -111,8 +105,9 @@ var _ = Describe("Common registry users:", func() { subjectRef := RegistryRef(Host, testRepo, foobar.Tag) prepare(RegistryRef(Host, ImageRepo, foobar.Tag), subjectRef) // test - MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() - + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). + WithWorkDir(tempDir). + MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() // validate var index ocispec.Index bytes := ORAS("discover", subjectRef, "-o", "json").Exec().Out.Contents() diff --git a/test/e2e/suite/command/discover.go b/test/e2e/suite/command/discover.go index c753edfce..22d717d02 100644 --- a/test/e2e/suite/command/discover.go +++ b/test/e2e/suite/command/discover.go @@ -262,7 +262,7 @@ var _ = Describe("OCI image layout users:", func() { When("running discover command with table output", func() { format := "table" - It("should direct referrers of a subject", func() { + It("should get direct referrers of a subject", func() { referrers := []ocispec.Descriptor{foobar.SBOMImageReferrer, foobar.SBOMArtifactReferrer} // prepare subjectRef := LayoutRef(GinkgoT().TempDir(), foobar.Tag) From e068e8da7aa889337e6cc7b6ce8e7b02663bc539 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 04:46:42 +0000 Subject: [PATCH 05/23] use pack type Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 20 +++++++++++++------- cmd/oras/root/push.go | 3 ++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index f403f9813..fe1059e3e 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -19,30 +19,36 @@ import ( "fmt" "github.com/spf13/pflag" + "oras.land/oras-go/v2" ) const ( - V1_1 = "1.1" + v1_1 = "1.1" V1_0 = "1.0" ) // ImageSpec option struct. type ImageSpec struct { - Flag string + flag string + PackType oras.PackManifestType } // Parse parses flags into the option. func (opts *ImageSpec) Parse() error { - switch opts.Flag { - case V1_1, V1_0: - return nil + switch opts.flag { + case v1_1: + opts.PackType = oras.PackManifestTypeImageV1_1_0_RC4 + case V1_0: + opts.PackType = oras.PackManifestTypeImageV1_1_0_RC2 + default: + return fmt.Errorf("unknown image specification flag: %q", opts.flag) } - return fmt.Errorf("unknown image specification flag: %q", opts.Flag) + return nil } // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.Flag, "image-spec", V1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", V1_1, V1_0)) + fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, V1_0)) } // distributionSpec option struct. diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index b0d95bfd4..bd6467b05 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -107,7 +107,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return err } - if opts.ImageSpec.Flag == option.V1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { + if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC2 && opts.manifestConfigRef != "" && opts.artifactType != "" { return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") } return nil @@ -137,6 +137,7 @@ func runPush(ctx context.Context, opts pushOptions) error { ConfigAnnotations: annotations[option.AnnotationConfig], ManifestAnnotations: annotations[option.AnnotationManifest], PackImageManifest: true, + PackManifestType: opts.ImageSpec.PackType, } store, err := file.New("") if err != nil { From 9520375ba4228f8929616f2f11c9c0ad74848eba Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 8 Aug 2023 12:52:51 +0000 Subject: [PATCH 06/23] update e2e go mod Signed-off-by: Billy Zha --- test/e2e/go.mod | 4 ++-- test/e2e/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 09b86ff2c..9f1bee561 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -6,9 +6,9 @@ require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0-rc4 gopkg.in/yaml.v2 v2.4.0 - oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b + oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 ) require ( diff --git a/test/e2e/go.sum b/test/e2e/go.sum index bac81e704..350be0911 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -20,8 +20,8 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -47,5 +47,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b h1:NYynybpqtG3lLTZMWNlrvUlcyGakCke57tg4TX6w2kA= -oras.land/oras-go/v2 v2.2.1-0.20230627113607-6b5bd4b4372b/go.mod h1:goptA58HogB/6sLN7KV6FgoiurcVxd5QBqv8wMVB0as= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542 h1:lK/VtXN9+dzSD3EMuCXu0D28OJadBxB9yUpaJBDIABM= +oras.land/oras-go/v2 v2.2.1-0.20230807082644-bbe92af00542/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ= From 09755f2df6ea527ed77d450c6ce4b16ac4182101 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 01:09:06 +0000 Subject: [PATCH 07/23] remove PackManifestTypeImageV1_1_0_RC2 Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 11 ++++++----- cmd/oras/root/push.go | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index fe1059e3e..625a1e175 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -23,8 +23,9 @@ import ( ) const ( - v1_1 = "1.1" - V1_0 = "1.0" + v1_1 = "1.1" + v1_0 = "1.0" + PackManifestTypeImageV1_0 = 0 ) // ImageSpec option struct. @@ -38,8 +39,8 @@ func (opts *ImageSpec) Parse() error { switch opts.flag { case v1_1: opts.PackType = oras.PackManifestTypeImageV1_1_0_RC4 - case V1_0: - opts.PackType = oras.PackManifestTypeImageV1_1_0_RC2 + case v1_0: + opts.PackType = PackManifestTypeImageV1_0 default: return fmt.Errorf("unknown image specification flag: %q", opts.flag) } @@ -48,7 +49,7 @@ func (opts *ImageSpec) Parse() error { // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, V1_0)) + fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, v1_0)) } // distributionSpec option struct. diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index bd6467b05..a1e93936b 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -107,7 +107,7 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t return err } - if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC2 && opts.manifestConfigRef != "" && opts.artifactType != "" { + if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") } return nil From e298234bd4beb5409b1f6add7a8f75f4b04ab62f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 03:22:06 +0000 Subject: [PATCH 08/23] fix artifact push without subject Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index a1e93936b..52bbe1309 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -155,6 +155,12 @@ func runPush(ctx context.Context, opts pushOptions) error { } desc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &desc + } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { + configDesc := ocispec.DescriptorEmptyJSON + configDesc.MediaType = oras.MediaTypeUnknownConfig + //store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) + configDesc.Annotations = packOpts.ConfigAnnotations + packOpts.ConfigDescriptor = &configDesc } descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { From 3e0390603808af70c5308a7e442eba4784639fcd Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 03:27:13 +0000 Subject: [PATCH 09/23] add data Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 52bbe1309..fd59cb1e5 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -16,6 +16,7 @@ limitations under the License. package root import ( + "bytes" "context" "errors" "fmt" @@ -158,7 +159,7 @@ func runPush(ctx context.Context, opts pushOptions) error { } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { configDesc := ocispec.DescriptorEmptyJSON configDesc.MediaType = oras.MediaTypeUnknownConfig - //store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) + store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) configDesc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &configDesc } From 9e801c53370e987c273c10c3deed1f3435d67833 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 9 Aug 2023 03:32:29 +0000 Subject: [PATCH 10/23] check returned error Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index fd59cb1e5..fc60c4ad3 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -159,7 +159,9 @@ func runPush(ctx context.Context, opts pushOptions) error { } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { configDesc := ocispec.DescriptorEmptyJSON configDesc.MediaType = oras.MediaTypeUnknownConfig - store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)) + if err = store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)); err != nil { + return err + } configDesc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &configDesc } From 93b66b80956a6b5ef42db08af0874028fdce39bf Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 00:57:57 +0000 Subject: [PATCH 11/23] default to unknown artifact type when not provided Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 5 +++-- cmd/oras/root/push.go | 21 +++++++-------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index 625a1e175..988fa8e02 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -23,8 +23,9 @@ import ( ) const ( - v1_1 = "1.1" - v1_0 = "1.0" + v1_1 = "v1.1" + v1_0 = "v1.0" + // TODO: pending on https://github.com/oras-project/oras-go/issues/568 PackManifestTypeImageV1_0 = 0 ) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index fc60c4ad3..8239d06e0 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -16,7 +16,6 @@ limitations under the License. package root import ( - "bytes" "context" "errors" "fmt" @@ -103,16 +102,18 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t opts.RawReference = refs[0] opts.extraRefs = refs[1:] opts.FileRefs = args[1:] - if err := option.Parse(&opts); err != nil { return err } - - if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 { + if opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + } + if opts.manifestConfigRef == "" && opts.artifactType == "" { + opts.artifactType = oras.MediaTypeUnknownArtifact + } } return nil - }, RunE: func(cmd *cobra.Command, args []string) error { return runPush(cmd.Context(), opts) @@ -156,14 +157,6 @@ func runPush(ctx context.Context, opts pushOptions) error { } desc.Annotations = packOpts.ConfigAnnotations packOpts.ConfigDescriptor = &desc - } else if opts.ImageSpec.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.artifactType == "" { - configDesc := ocispec.DescriptorEmptyJSON - configDesc.MediaType = oras.MediaTypeUnknownConfig - if err = store.Push(ctx, configDesc, bytes.NewReader(configDesc.Data)); err != nil { - return err - } - configDesc.Annotations = packOpts.ConfigAnnotations - packOpts.ConfigDescriptor = &configDesc } descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { From 2117e45a0a1ed59a07f6907fd7a6d7c4a4c07af0 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:03:51 +0000 Subject: [PATCH 12/23] fix pack type Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 8239d06e0..edaa4b774 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -105,13 +105,11 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t if err := option.Parse(&opts); err != nil { return err } - if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 { - if opts.manifestConfigRef != "" && opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") - } - if opts.manifestConfigRef == "" && opts.artifactType == "" { - opts.artifactType = oras.MediaTypeUnknownArtifact - } + if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + } + if opts.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.manifestConfigRef == "" && opts.artifactType == "" { + opts.artifactType = oras.MediaTypeUnknownArtifact } return nil }, From 64b3ab588a92b4b893465f12e1d3849cff56cdca Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:12:37 +0000 Subject: [PATCH 13/23] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/push.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/suite/command/push.go b/test/e2e/suite/command/push.go index 17139fed4..12fd16ecc 100644 --- a/test/e2e/suite/command/push.go +++ b/test/e2e/suite/command/push.go @@ -47,7 +47,7 @@ var _ = Describe("Remote registry users:", func() { When("pushing to registy without OCI artifact support", func() { repoPrefix := fmt.Sprintf("command/push/%d", GinkgoRandomSeed()) statusKeys := []match.StateKey{ - foobar.ImageConfigStateKey("application/vnd.unknown.config.v1+json"), + foobar.ImageConfigStateKey("application/vnd.unknown.artifact.v1"), foobar.FileBarStateKey, } It("should push files without customized media types", func() { @@ -230,7 +230,7 @@ var _ = Describe("OCI image layout users:", func() { tag := "e2e" When("pushing to registy without OCI artifact support", func() { statusKeys := []match.StateKey{ - foobar.ImageConfigStateKey("application/vnd.unknown.config.v1+json"), + foobar.ImageConfigStateKey("application/vnd.oci.empty.v1+json"), foobar.FileBarStateKey, } From 552f65545b3827da81475a86f62efb26ca6eb75a Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:22:34 +0000 Subject: [PATCH 14/23] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite/command/push.go b/test/e2e/suite/command/push.go index 12fd16ecc..0fe788a79 100644 --- a/test/e2e/suite/command/push.go +++ b/test/e2e/suite/command/push.go @@ -47,7 +47,7 @@ var _ = Describe("Remote registry users:", func() { When("pushing to registy without OCI artifact support", func() { repoPrefix := fmt.Sprintf("command/push/%d", GinkgoRandomSeed()) statusKeys := []match.StateKey{ - foobar.ImageConfigStateKey("application/vnd.unknown.artifact.v1"), + foobar.ImageConfigStateKey("application/vnd.oci.empty.v1+json"), foobar.FileBarStateKey, } It("should push files without customized media types", func() { From 34da8f6ae2071418f70502f15e6254ffd75c032f Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:37:03 +0000 Subject: [PATCH 15/23] remove examples Signed-off-by: Billy Zha --- cmd/oras/root/attach.go | 4 ---- cmd/oras/root/push.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index a18be3e0d..c539eb336 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -54,10 +54,6 @@ func attachCmd() *cobra.Command { Example - Attach file 'hi.txt' with type 'doc/example' to manifest 'hello:v1' in registry 'localhost:5000': oras attach --artifact-type doc/example localhost:5000/hello:v1 hi.txt -Example - Attach file "hi.txt" with specific media type when building the manifest: - oras attach --artifact-type doc/example --image-spec v1.1-image localhost:5000/hello:v1 hi.txt # OCI image - oras attach --artifact-type doc/example --image-spec v1.1-artifact localhost:5000/hello:v1 hi.txt # OCI artifact - Example - Attach file "hi.txt" using a specific method for the Referrers API: oras attach --artifact-type doc/example --distribution-spec v1.1-referrers-api localhost:5000/hello:v1 hi.txt # via API oras attach --artifact-type doc/example --distribution-spec v1.1-referrers-tag localhost:5000/hello:v1 hi.txt # via tag scheme diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index edaa4b774..0402f17ad 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -71,10 +71,6 @@ Example - Push file "hi.txt" with config type "application/vnd.me.config": Example - Push file "hi.txt" with the custom manifest config "config.json" of the custom media type "application/vnd.me.config": oras push --config config.json:application/vnd.me.config localhost:5000/hello:v1 hi.txt -Example - Push file "hi.txt" with specific media type when building the manifest: - oras push --image-spec v1.1-image localhost:5000/hello:v1 hi.txt # OCI image - oras push --image-spec v1.1-artifact localhost:5000/hello:v1 hi.txt # OCI artifact - Example - Push file to the insecure registry: oras push --insecure localhost:5000/hello:v1 hi.txt From 88fd27259427289125bf8124a8a4730fa481fac2 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 01:42:59 +0000 Subject: [PATCH 16/23] fix e2e Signed-off-by: Billy Zha --- test/e2e/suite/command/attach.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite/command/attach.go b/test/e2e/suite/command/attach.go index 9071bccee..b0e5fba68 100644 --- a/test/e2e/suite/command/attach.go +++ b/test/e2e/suite/command/attach.go @@ -170,7 +170,7 @@ var _ = Describe("Fallback registry users:", func() { subjectRef := RegistryRef(FallbackHost, testRepo, foobar.Tag) prepare(RegistryRef(FallbackHost, ArtifactRepo, foobar.Tag), subjectRef) // test - ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia), "--image-spec", "v1.1-image"). + ORAS("attach", "--artifact-type", "test.attach", subjectRef, fmt.Sprintf("%s:%s", foobar.AttachFileName, foobar.AttachFileMedia)). WithWorkDir(tempDir). MatchStatus([]match.StateKey{foobar.AttachFileStateKey}, false, 1).Exec() From 7cc3bf80acf5e996da42a3bcc9e2d7ea1e6aa29e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 16:06:58 +0000 Subject: [PATCH 17/23] add index support Signed-off-by: Billy Zha --- internal/graph/graph.go | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 77c054a59..bd6fe09ab 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -83,6 +83,19 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr } nodes = manifest.Blobs subject = manifest.Subject + + case ocispec.MediaTypeImageIndex: + var fetched []byte + fetched, err = content.FetchAll(ctx, fetcher, node) + if err != nil { + return + } + var index ocispec.Index + if err = json.Unmarshal(fetched, &index); err != nil { + return + } + nodes = index.Manifests + subject = index.Subject default: nodes, err = content.Successors(ctx, fetcher, node) } @@ -137,8 +150,25 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc if image.Subject == nil || !content.Equal(*image.Subject, desc) { continue } - node.ArtifactType = image.Config.MediaType + node.ArtifactType = image.ArtifactType + if node.ArtifactType == "" { + node.ArtifactType = image.Config.MediaType + } node.Annotations = image.Annotations + case ocispec.MediaTypeImageIndex: + fetched, err := fetchBytes(ctx, target, node) + if err != nil { + return nil, err + } + var index ocispec.Index + if err := json.Unmarshal(fetched, &index); err != nil { + return nil, err + } + if index.Subject == nil || !content.Equal(*index.Subject, desc) { + continue + } + node.ArtifactType = index.ArtifactType + node.Annotations = index.Annotations default: continue } From 788398b4b96bfcb857e951a5ddcb13967316556c Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 16:07:56 +0000 Subject: [PATCH 18/23] resolve comment Signed-off-by: Billy Zha --- cmd/oras/internal/option/spec.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/oras/internal/option/spec.go b/cmd/oras/internal/option/spec.go index 988fa8e02..aa08fcae6 100644 --- a/cmd/oras/internal/option/spec.go +++ b/cmd/oras/internal/option/spec.go @@ -23,8 +23,8 @@ import ( ) const ( - v1_1 = "v1.1" - v1_0 = "v1.0" + ImageSpecV1_1 = "v1.1" + ImageSpecV1_0 = "v1.0" // TODO: pending on https://github.com/oras-project/oras-go/issues/568 PackManifestTypeImageV1_0 = 0 ) @@ -38,9 +38,9 @@ type ImageSpec struct { // Parse parses flags into the option. func (opts *ImageSpec) Parse() error { switch opts.flag { - case v1_1: + case ImageSpecV1_1: opts.PackType = oras.PackManifestTypeImageV1_1_0_RC4 - case v1_0: + case ImageSpecV1_0: opts.PackType = PackManifestTypeImageV1_0 default: return fmt.Errorf("unknown image specification flag: %q", opts.flag) @@ -50,7 +50,7 @@ func (opts *ImageSpec) Parse() error { // ApplyFlags applies flags to a command flag set. func (opts *ImageSpec) ApplyFlags(fs *pflag.FlagSet) { - fs.StringVar(&opts.flag, "image-spec", v1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", v1_1, v1_0)) + fs.StringVar(&opts.flag, "image-spec", ImageSpecV1_1, fmt.Sprintf("[Experimental] specify manifest type for building artifact. options: %s, %s", ImageSpecV1_1, ImageSpecV1_0)) } // distributionSpec option struct. From 93d7a8a4e580a8a99d8339c799e1e3739db4218e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Thu, 10 Aug 2023 16:31:52 +0000 Subject: [PATCH 19/23] remove duplicated function Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 5 +- internal/graph/graph.go | 40 --------------- internal/graph/graph_test.go | 97 ------------------------------------ 3 files changed, 4 insertions(+), 138 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index f8a953b39..224de53f7 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -25,6 +25,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" "oras.land/oras/internal/graph" @@ -117,7 +118,9 @@ func runCopy(ctx context.Context, opts copyOptions) error { committed := &sync.Map{} extendedCopyOptions := oras.DefaultExtendedCopyOptions extendedCopyOptions.Concurrency = opts.concurrency - extendedCopyOptions.FindPredecessors = graph.FindReferrerPredecessors + extendedCopyOptions.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + return graph.Referrers(ctx, src, desc, "") + } extendedCopyOptions.PreCopy = display.StatusPrinter("Copying", opts.Verbose) extendedCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index bd6fe09ab..086a51447 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -179,46 +179,6 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc return results, nil } -// FindReferrerPredecessors returns referrer nodes of desc in target. -func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - var results []ocispec.Descriptor - if repo, ok := src.(registry.ReferrerLister); ok { - // get referrers directly - err := repo.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error { - results = append(results, referrers...) - return nil - }) - if err != nil { - return nil, err - } - return results, nil - } - predecessors, err := src.Predecessors(ctx, desc) - if err != nil { - return nil, err - } - for _, node := range predecessors { - switch node.MediaType { - case MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: - results = append(results, node) - case ocispec.MediaTypeImageIndex: - fetched, err := fetchBytes(ctx, src, node) - if err != nil { - return nil, err - } - // convert to json - var index ocispec.Index - if err := json.Unmarshal(fetched, &index); err != nil { - return nil, err - } - if index.Subject != nil && content.Equal(*index.Subject, desc) { - results = append(results, node) - } - } - } - return results, nil -} - func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]byte, error) { rc, err := fetcher.Fetch(ctx, desc) if err != nil { diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 370d34bb2..d33c4ad8c 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -39,14 +39,6 @@ func (e *errLister) Referrers(ctx context.Context, desc ocispec.Descriptor, arti return errors.New("") } -type errFinder struct { - oras.ReadOnlyGraphTarget -} - -func (e *errFinder) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { - return nil, errors.New("") -} - type refLister struct { referrers []ocispec.Descriptor oras.ReadOnlyGraphTarget @@ -265,92 +257,3 @@ func TestSuccessors(t *testing.T) { }) } } - -func TestFindReferrerPredecessors(t *testing.T) { - ctx := context.Background() - var blobs [][]byte - var descs []ocispec.Descriptor - appendBlob := func(mediaType string, blob []byte) { - blobs = append(blobs, blob) - descs = append(descs, ocispec.Descriptor{ - MediaType: mediaType, - Digest: digest.FromBytes(blob), - Size: int64(len(blob)), - }) - } - generateImage := func(subject *ocispec.Descriptor, annotations map[string]string, config ocispec.Descriptor, layers ...ocispec.Descriptor) { - manifest := ocispec.Manifest{ - Subject: subject, - Config: config, - Layers: layers, - Annotations: annotations, - } - manifestJSON, err := json.Marshal(manifest) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) - } - generateIndex := func(manifests ...ocispec.Descriptor) { - index := ocispec.Index{ - Manifests: manifests, - } - manifestJSON, err := json.Marshal(index) - if err != nil { - t.Fatal(err) - } - appendBlob(ocispec.MediaTypeImageIndex, manifestJSON) - } - const ( - subject = iota - imgConfig - image - ) - var anno map[string]string - appendBlob(ocispec.MediaTypeImageLayer, []byte("blob")) - imageType := "test.image" - appendBlob(imageType, []byte("config content")) - generateImage(&descs[subject], anno, descs[imgConfig]) - imageDesc := descs[image] - imageDesc.Annotations = anno - imageDesc.ArtifactType = imageType - generateIndex(descs[subject]) - - referrers := []ocispec.Descriptor{descs[image], descs[image]} - memory := memory.New() - for i := range descs { - if err := memory.Push(ctx, descs[i], bytes.NewReader(blobs[i])); err != nil { - t.Errorf("Error pushing %v\n", err) - } - } - finder := &predecessorFinder{Store: memory} - type args struct { - ctx context.Context - src content.ReadOnlyGraphStorage - desc ocispec.Descriptor - } - tests := []struct { - name string - args args - want []ocispec.Descriptor - wantErr bool - }{ - {"should failed to get referrers", args{ctx, &errLister{}, ocispec.Descriptor{}}, nil, true}, - {"should failed to get predecessor", args{ctx, &errFinder{}, ocispec.Descriptor{}}, nil, true}, - {"should return referrers when target is a referrer lister", args{ctx, &refLister{referrers: referrers}, ocispec.Descriptor{}}, referrers, false}, - {"should return image for config node", args{ctx, finder, descs[imgConfig]}, []ocispec.Descriptor{descs[image]}, false}, - {"should return image for subject node", args{ctx, finder, descs[subject]}, []ocispec.Descriptor{descs[image]}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := FindReferrerPredecessors(tt.args.ctx, tt.args.src, tt.args.desc) - if (err != nil) != tt.wantErr { - t.Errorf("FindReferrerPredecessors() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("FindReferrerPredecessors() = %v, want %v", got, tt.want) - } - }) - } -} From fc8d263e97a73055a1e3b8883a9335e3c836b094 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Mon, 14 Aug 2023 00:40:45 +0000 Subject: [PATCH 20/23] resolve comment Signed-off-by: Billy Zha --- cmd/oras/root/push.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index 0402f17ad..206a742b7 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -101,11 +101,15 @@ Example - Push file "hi.txt" into an OCI image layout folder 'layout-dir' with t if err := option.Parse(&opts); err != nil { return err } - if opts.ImageSpec.PackType == option.PackManifestTypeImageV1_0 && opts.manifestConfigRef != "" && opts.artifactType != "" { - return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") - } - if opts.PackType == oras.PackManifestTypeImageV1_1_0_RC4 && opts.manifestConfigRef == "" && opts.artifactType == "" { - opts.artifactType = oras.MediaTypeUnknownArtifact + switch opts.PackType { + case option.PackManifestTypeImageV1_0: + if opts.manifestConfigRef != "" && opts.artifactType != "" { + return errors.New("--artifact-type and --config cannot both be provided for 1.0 OCI image") + } + case oras.PackManifestTypeImageV1_1_0_RC4: + if opts.manifestConfigRef == "" && opts.artifactType == "" { + opts.artifactType = oras.MediaTypeUnknownArtifact + } } return nil }, From a854802fb19de6b5f0efd22694c85c9c15e7b477 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Mon, 14 Aug 2023 02:50:35 +0000 Subject: [PATCH 21/23] resolve comment Signed-off-by: Billy Zha --- internal/graph/graph.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 086a51447..898372814 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -83,7 +83,6 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr } nodes = manifest.Blobs subject = manifest.Subject - case ocispec.MediaTypeImageIndex: var fetched []byte fetched, err = content.FetchAll(ctx, fetcher, node) From 240dc6baf63e23bfb19b3c90936d27cb0fb46bc5 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 18 Aug 2023 01:32:28 +0000 Subject: [PATCH 22/23] resolve comment Signed-off-by: Billy Zha --- internal/graph/graph.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 898372814..78c7ded90 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -171,7 +171,9 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc default: continue } - if node.ArtifactType != "" && (artifactType == "" || artifactType == node.ArtifactType) { + if artifactType == "" || artifactType == node.ArtifactType { + // artifactType is allowed to be empty & empty will pass the filtering by default + // https://github.com/opencontainers/distribution-spec/issues/458 results = append(results, node) } } From 0c382f425c130d0ae8858d94cbcc5b9b168af238 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 18 Aug 2023 03:45:55 +0000 Subject: [PATCH 23/23] resolve comment Signed-off-by: Billy Zha --- internal/graph/graph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 78c7ded90..4269d36df 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -172,7 +172,7 @@ func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc oc continue } if artifactType == "" || artifactType == node.ArtifactType { - // artifactType is allowed to be empty & empty will pass the filtering by default + // the field artifactType in referrers descriptor is allowed to be empty // https://github.com/opencontainers/distribution-spec/issues/458 results = append(results, node) }