diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 90f7cb090..908493618 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -17,6 +17,7 @@ package root import ( "context" + "encoding/json" "fmt" "strings" "sync" @@ -28,6 +29,7 @@ import ( "oras.land/oras-go/v2/content" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/docker" "oras.land/oras/internal/graph" ) @@ -137,28 +139,19 @@ func runCopy(ctx context.Context, opts copyOptions) error { var desc ocispec.Descriptor rOpts := oras.DefaultResolveOptions rOpts.TargetPlatform = opts.Platform.Platform - if dstRef := opts.To.Reference; dstRef == "" { + if opts.recursive { desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts) if err != nil { return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) } - if opts.recursive { - err = oras.ExtendedCopyGraph(ctx, src, dst, desc, extendedCopyOptions.ExtendedCopyGraphOptions) - } else { - err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions) - } + err = recursiveCopy(ctx, src, dst, opts.To.Reference, desc, extendedCopyOptions) } else { - if opts.recursive { - srcRef := opts.From.Reference - if rOpts.TargetPlatform != nil { - // resolve source reference to specified platform - desc, err := oras.Resolve(ctx, src, opts.From.Reference, rOpts) - if err != nil { - return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) - } - srcRef = desc.Digest.String() + if opts.To.Reference == "" { + desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts) + if err != nil { + return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) } - desc, err = oras.ExtendedCopy(ctx, src, srcRef, dst, dstRef, extendedCopyOptions) + err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions) } else { copyOptions := oras.CopyOptions{ CopyGraphOptions: extendedCopyOptions.CopyGraphOptions, @@ -166,7 +159,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { if opts.Platform.Platform != nil { copyOptions.WithTargetPlatform(opts.Platform.Platform) } - desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, dstRef, copyOptions) + desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, opts.To.Reference, copyOptions) } } if err != nil { @@ -191,3 +184,44 @@ func runCopy(ctx context.Context, opts copyOptions) error { return nil } + +// recursiveCopy copies an artifact and its referrers from one target to another. +// If the artifact is a manifest list or index, referrers of its manifests are copied as well. +func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error { + if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList { + fetched, err := content.FetchAll(ctx, src, root) + if err != nil { + return err + } + var index ocispec.Index + if err = json.Unmarshal(fetched, &index); err != nil { + return nil + } + + referrers, err := graph.FindPredecessors(ctx, src, index.Manifests, opts) + if err != nil { + return err + } + + findPredecessor := opts.FindPredecessors + opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + descs, err := findPredecessor(ctx, src, desc) + if err != nil { + return nil, err + } + if content.Equal(desc, root) { + // make sure referrers of child manifests are copied by pointing them to root + descs = append(descs, referrers...) + } + return descs, nil + } + } + + var err error + if dstRef == "" || dstRef == root.Digest.String() { + err = oras.ExtendedCopyGraph(ctx, src, dst, root, opts.ExtendedCopyGraphOptions) + } else { + _, err = oras.ExtendedCopy(ctx, src, root.Digest.String(), dst, dstRef, opts) + } + return err +} diff --git a/internal/docker/mediatype.go b/internal/docker/mediatype.go index 5f07ff54f..4ba809fea 100644 --- a/internal/docker/mediatype.go +++ b/internal/docker/mediatype.go @@ -17,5 +17,6 @@ package docker // docker media types const ( - MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" ) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 4269d36df..e5323124d 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -18,8 +18,11 @@ package graph import ( "context" "encoding/json" + "sync" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" + "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/registry" "oras.land/oras/internal/docker" @@ -188,3 +191,31 @@ func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descr defer rc.Close() return content.ReadAll(rc, desc) } + +// FindPredecessors returns all predecessors of descs in src concurrently. +func FindPredecessors(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) { + var referrers []ocispec.Descriptor + g, ctx := errgroup.WithContext(ctx) + var m sync.Mutex + if opts.Concurrency != 0 { + g.SetLimit(opts.Concurrency) + } + for _, desc := range descs { + g.Go(func(node ocispec.Descriptor) func() error { + return func() error { + descs, err := opts.FindPredecessors(ctx, src, node) + if err != nil { + return err + } + m.Lock() + defer m.Unlock() + referrers = append(referrers, descs...) + return nil + } + }(desc)) + } + if err := g.Wait(); err != nil { + return nil, err + } + return referrers, nil +} diff --git a/test/e2e/internal/testdata/multi_arch/const.go b/test/e2e/internal/testdata/multi_arch/const.go index 119764386..5915b5942 100644 --- a/test/e2e/internal/testdata/multi_arch/const.go +++ b/test/e2e/internal/testdata/multi_arch/const.go @@ -23,6 +23,7 @@ import ( var ( Tag = "multi" + EmptyTag = "empty_index" Digest = "sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f" Manifest = `{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1","size":458,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c","size":458,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255","size":458,"platform":{"architecture":"arm","os":"linux","variant":"v7"}}]}` Descriptor = `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f","size":706}` diff --git a/test/e2e/suite/command/cp.go b/test/e2e/suite/command/cp.go index 317d0f989..5e0893d73 100644 --- a/test/e2e/suite/command/cp.go +++ b/test/e2e/suite/command/cp.go @@ -115,7 +115,7 @@ var _ = Describe("1.1 registry users:", func() { }) It("should copy an image and its referrers to a new repository", func() { - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag) dst := RegistryRef(ZOTHost, cpTestRepo("referrers"), foobar.Digest) ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() @@ -123,7 +123,7 @@ var _ = Describe("1.1 registry users:", func() { }) It("should copy a multi-arch image and its referrers to a new repository via tag", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) dstRepo := cpTestRepo("index-referrers") dst := RegistryRef(ZOTHost, dstRepo, "copiedTag") @@ -142,13 +142,47 @@ var _ = Describe("1.1 registry users:", func() { Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())). - WithDescription("not copy referrer of successor"). - ExpectFailure(). + WithDescription("copy referrer of successor"). + Exec() + }) + + It("should copy a multi-arch image and its referrers without concurrency limitation", func() { + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) + src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) + dstRepo := cpTestRepo("index-referrers-concurrent") + dst := RegistryRef(ZOTHost, dstRepo, "copiedTag") + // test + ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0"). + MatchStatus(stateKeys, true, len(stateKeys)). + MatchKeyWords("Digest: " + ma.Digest). + Exec() + // validate + CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.Digest), dst) + var index ocispec.Index + bytes := ORAS("discover", dst, "-o", "json", "--artifact-type", ma.IndexReferrerConfigStateKey.Name). + MatchKeyWords(ma.IndexReferrerDigest). + WithDescription("copy image referrer"). + Exec().Out.Contents() + Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) + Expect(len(index.Manifests)).To(Equal(1)) + Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) + ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())). + WithDescription("copy referrer of successor"). Exec() }) + It("should copy an empty index", func() { + src := RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag) + dstRepo := cpTestRepo("empty-index") + dst := RegistryRef(ZOTHost, dstRepo, "copiedTag") + // test + ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0").Exec() + // validate + CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag), dst) + }) + It("should copy a multi-arch image and its referrers to a new repository via digest", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) dstRepo := cpTestRepo("index-referrers-digest") dst := RegistryRef(ZOTHost, dstRepo, ma.Digest) @@ -168,7 +202,6 @@ var _ = Describe("1.1 registry users:", func() { Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())). WithDescription("not copy referrer of successor"). - ExpectFailure(). Exec() }) @@ -270,7 +303,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() { When("running `cp`", func() { It("should copy an image artifact and its referrers from a registry to a fallback registry", func() { repo := cpTestRepo("to-fallback") - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String()) dst := RegistryRef(FallbackHost, repo, "") ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() @@ -280,7 +313,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() { }) It("should copy an image artifact and its referrers from a fallback registry to a registry", func() { repo := cpTestRepo("from-fallback") - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(FallbackHost, ArtifactRepo, foobar.SBOMImageReferrer.Digest.String()) dst := RegistryRef(ZOTHost, repo, "") ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() @@ -439,7 +472,7 @@ var _ = Describe("OCI layout users:", func() { }) It("should copy a tagged image and its referrers from a registry to an OCI image layout", func() { - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) dst := LayoutRef(GinkgoT().TempDir(), "copied") src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag) // test @@ -451,7 +484,7 @@ var _ = Describe("OCI layout users:", func() { }) It("should copy a image and its referrers from a registry to an OCI image layout via digest", func() { - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) toDir := GinkgoT().TempDir() src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Digest) // test @@ -463,7 +496,7 @@ var _ = Describe("OCI layout users:", func() { }) It("should copy a multi-arch image and its referrers from a registry to an OCI image layout a via tag", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) toDir := GinkgoT().TempDir() dst := LayoutRef(toDir, "copied") @@ -485,13 +518,12 @@ var _ = Describe("OCI layout users:", func() { Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) ORAS("manifest", "fetch", Flags.Layout, LayoutRef(toDir, ma.LinuxAMD64Referrer.Digest.String())). - WithDescription("not copy referrer of successor"). - ExpectFailure(). + WithDescription("copy referrer of successor"). Exec() }) It("should copy a multi-arch image and its referrers from an OCI image layout to a registry via digest", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) fromDir := GinkgoT().TempDir() src := LayoutRef(fromDir, ma.Tag) dst := RegistryRef(ZOTHost, cpTestRepo("recursive-from-layout"), "copied") @@ -514,9 +546,8 @@ var _ = Describe("OCI layout users:", func() { Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) - ORAS("manifest", "fetch", LayoutRef(fromDir, ma.LinuxAMD64Referrer.Digest.String())). - WithDescription("not copy referrer of successor"). - ExpectFailure(). + ORAS("manifest", "fetch", dst). + WithDescription("copy referrer of successor"). Exec() }) diff --git a/test/e2e/suite/command/pull.go b/test/e2e/suite/command/pull.go index 8bf6c1225..65d5e6eea 100644 --- a/test/e2e/suite/command/pull.go +++ b/test/e2e/suite/command/pull.go @@ -122,7 +122,7 @@ var _ = Describe("OCI spec 1.1 registry users:", func() { It("should copy an artifact with blob", func() { repo := cpTestRepo("artifact-with-blob") - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String()) dst := RegistryRef(FallbackHost, repo, "") ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() diff --git a/test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 b/test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 new file mode 100644 index 000000000..3df36494f --- /dev/null +++ b/test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 @@ -0,0 +1 @@ +{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[]} diff --git a/test/e2e/testdata/zot/command/images/index.json b/test/e2e/testdata/zot/command/images/index.json index 4b2b1af71..2162bc883 100644 --- a/test/e2e/testdata/zot/command/images/index.json +++ b/test/e2e/testdata/zot/command/images/index.json @@ -44,6 +44,14 @@ "architecture": "amd64", "os": "linux" } + }, + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286", + "size": 89, + "annotations": { + "org.opencontainers.image.ref.name": "empty_index" + } } ] } \ No newline at end of file