From ed9d5843b12ee57b3f91fb7196f11cc80842f779 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Wed, 2 Nov 2022 21:26:53 +0800 Subject: [PATCH] feat!: support OCI image fallback for `oras push` and `oras attach` (#665) Since oras-go will take care of fallback with tag schema if referrer API not supported, this PR provides OCI Image fallback when-and-only-when OCI artifact is not supported. Resolves: #654 Signed-off-by: Billy Zha --- cmd/oras/attach.go | 55 +++++--------- cmd/oras/push.go | 180 +++++++++++++++++++++++++++++++++------------ go.mod | 2 +- go.sum | 4 +- 4 files changed, 155 insertions(+), 86 deletions(-) diff --git a/cmd/oras/attach.go b/cmd/oras/attach.go index 980643431..af10f36f5 100644 --- a/cmd/oras/attach.go +++ b/cmd/oras/attach.go @@ -19,14 +19,12 @@ import ( "context" "errors" "fmt" - "sync" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content/file" - "oras.land/oras/cmd/oras/internal/display" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/option" ) @@ -113,42 +111,33 @@ func runAttach(opts attachOptions) error { if err != nil { return err } - root, err := oras.Pack( - ctx, store, opts.artifactType, descs, - oras.PackOptions{ - Subject: &subject, - ManifestAnnotations: annotations[option.AnnotationManifest], - }) - if err != nil { - return err - } // prepare push - committed := &sync.Map{} - graphCopyOptions := oras.DefaultCopyGraphOptions - graphCopyOptions.Concurrency = opts.concurrency - graphCopyOptions.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if isEqualOCIDescriptor(node, root) { - // skip subject - return descs, nil - } - return content.Successors(ctx, fetcher, node) + packOpts := oras.PackOptions{ + Subject: &subject, + ManifestAnnotations: annotations[option.AnnotationManifest], } - graphCopyOptions.PreCopy = display.StatusPrinter("Uploading", opts.Verbose) - graphCopyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { - committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - return display.PrintStatus(desc, "Exists ", opts.Verbose) + pack := func() (ocispec.Descriptor, error) { + return oras.Pack(ctx, store, opts.artifactType, descs, packOpts) } - graphCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { - committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, opts.Verbose); err != nil { - return err + + graphCopyOptions := oras.DefaultCopyGraphOptions + graphCopyOptions.Concurrency = opts.concurrency + updateDisplayOption(&graphCopyOptions, store, opts.Verbose) + copy := func(root ocispec.Descriptor) error { + if root.MediaType == ocispec.MediaTypeArtifactManifest { + graphCopyOptions.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if content.Equal(node, root) { + // skip subject + return descs, nil + } + return content.Successors(ctx, fetcher, node) + } } - return display.PrintStatus(desc, "Uploaded ", opts.Verbose) + return oras.CopyGraph(ctx, store, dst, root, graphCopyOptions) } - // push - err = oras.CopyGraph(ctx, store, dst, root, graphCopyOptions) + root, err := pushArtifact(dst, pack, &packOpts, copy, &graphCopyOptions, opts.Verbose) if err != nil { return err } @@ -159,7 +148,3 @@ func runAttach(opts attachOptions) error { // Export manifest return opts.ExportManifest(ctx, store, root) } - -func isEqualOCIDescriptor(a, b ocispec.Descriptor) bool { - return a.Size == b.Size && a.Digest == b.Digest && a.MediaType == b.MediaType -} diff --git a/cmd/oras/push.go b/cmd/oras/push.go index f3ed32c08..3fbd8e785 100644 --- a/cmd/oras/push.go +++ b/cmd/oras/push.go @@ -17,8 +17,10 @@ package main import ( "context" + "encoding/json" "errors" "fmt" + "net/http" "strings" "sync" @@ -27,14 +29,12 @@ import ( "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/errcode" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" ) -const ( - tagStaged = "staged" -) - type pushOptions struct { option.Common option.Remote @@ -120,94 +120,178 @@ func runPush(opts pushOptions) error { return err } - // Prepare manifest + // prepare pack + packOpts := oras.PackOptions{ + ConfigAnnotations: annotations[option.AnnotationConfig], + ManifestAnnotations: annotations[option.AnnotationManifest], + } store := file.New("") defer store.Close() store.AllowPathTraversalOnWrite = opts.PathValidationDisabled - - // Ready to push - committed := &sync.Map{} - copyOptions := oras.DefaultCopyOptions - copyOptions.Concurrency = opts.concurrency - copyOptions.PreCopy = display.StatusPrinter("Uploading", opts.Verbose) - copyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { - committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - return display.PrintStatus(desc, "Exists ", opts.Verbose) - } - copyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { - committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, opts.Verbose); err != nil { + if opts.manifestConfigRef != "" { + path, cfgMediaType := parseFileReference(opts.manifestConfigRef, oras.MediaTypeUnknownConfig) + desc, err := store.Add(ctx, option.AnnotationConfig, cfgMediaType, path) + if err != nil { return err } - return display.PrintStatus(desc, "Uploaded ", opts.Verbose) + desc.Annotations = packOpts.ConfigAnnotations + packOpts.ConfigDescriptor = &desc + packOpts.PackImageManifest = true } - desc, err := packManifest(ctx, store, annotations, &opts) + descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) if err != nil { return err } + pack := func() (ocispec.Descriptor, error) { + root, err := oras.Pack(ctx, store, opts.artifactType, descs, packOpts) + if err != nil { + return ocispec.Descriptor{}, err + } + if err = store.Tag(ctx, root, root.Digest.String()); err != nil { + return ocispec.Descriptor{}, err + } + return root, nil + } - // Push + // prepare push dst, err := opts.NewRepository(opts.targetRef, opts.Common) if err != nil { return err } - if tag := dst.Reference.Reference; tag == "" { - err = oras.CopyGraph(ctx, store, dst, desc, copyOptions.CopyGraphOptions) - } else { - desc, err = oras.Copy(ctx, store, tagStaged, dst, tag, copyOptions) + copyOptions := oras.DefaultCopyOptions + copyOptions.Concurrency = opts.concurrency + updateDisplayOption(©Options.CopyGraphOptions, store, opts.Verbose) + copy := func(root ocispec.Descriptor) error { + if tag := dst.Reference.Reference; tag == "" { + err = oras.CopyGraph(ctx, store, dst, root, copyOptions.CopyGraphOptions) + } else { + _, err = oras.Copy(ctx, store, root.Digest.String(), dst, tag, copyOptions) + } + return err } + + // Push + root, err := pushArtifact(dst, pack, &packOpts, copy, ©Options.CopyGraphOptions, opts.Verbose) if err != nil { return err } - fmt.Println("Pushed", opts.targetRef) if len(opts.extraRefs) != 0 { - contentBytes, err := content.FetchAll(ctx, store, desc) + contentBytes, err := content.FetchAll(ctx, store, root) if err != nil { return err } tagBytesNOpts := oras.DefaultTagBytesNOptions tagBytesNOpts.Concurrency = opts.concurrency - if _, err = oras.TagBytesN(ctx, &display.TagManifestStatusPrinter{Repository: dst}, desc.MediaType, contentBytes, opts.extraRefs, tagBytesNOpts); err != nil { + if _, err = oras.TagBytesN(ctx, &display.TagManifestStatusPrinter{Repository: dst}, root.MediaType, contentBytes, opts.extraRefs, tagBytesNOpts); err != nil { return err } } - fmt.Println("Digest:", desc.Digest) + fmt.Println("Digest:", root.Digest) // Export manifest - return opts.ExportManifest(ctx, store, desc) + return opts.ExportManifest(ctx, store, root) } -func packManifest(ctx context.Context, store *file.Store, annotations map[string]map[string]string, opts *pushOptions) (ocispec.Descriptor, error) { - var packOpts oras.PackOptions - packOpts.ConfigAnnotations = annotations[option.AnnotationConfig] - packOpts.ManifestAnnotations = annotations[option.AnnotationManifest] - - if opts.manifestConfigRef != "" { - path, mediatype := parseFileReference(opts.manifestConfigRef, oras.MediaTypeUnknownConfig) - desc, err := store.Add(ctx, option.AnnotationConfig, mediatype, path) - if err != nil { - return ocispec.Descriptor{}, err +func updateDisplayOption(opts *oras.CopyGraphOptions, store content.Fetcher, verbose bool) { + committed := &sync.Map{} + opts.PreCopy = display.StatusPrinter("Uploading", verbose) + opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { + committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + return display.PrintStatus(desc, "Exists ", verbose) + } + opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { + committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, verbose); err != nil { + return err } - desc.Annotations = packOpts.ConfigAnnotations - packOpts.ConfigDescriptor = &desc - packOpts.PackImageManifest = true + return display.PrintStatus(desc, "Uploaded ", verbose) } - descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose) +} + +type packFunc func() (ocispec.Descriptor, error) +type copyFunc func(desc ocispec.Descriptor) error + +func pushArtifact(dst *remote.Repository, pack packFunc, packOpts *oras.PackOptions, copy copyFunc, copyOpts *oras.CopyGraphOptions, verbose bool) (ocispec.Descriptor, error) { + root, err := pack() if err != nil { return ocispec.Descriptor{}, err } - // pack artifact - manifestDesc, err := oras.Pack(ctx, store, opts.artifactType, descs, packOpts) + copyRootAttempted := false + preCopy := copyOpts.PreCopy + copyOpts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { + if content.Equal(root, desc) { + // copyRootAttempted helps track whether the returned error is + // generated from copying root. + copyRootAttempted = true + } + if preCopy != nil { + return preCopy(ctx, desc) + } + return nil + } + + // push + if err = copy(root); err == nil { + return root, nil + } + + if !copyRootAttempted || !isArtifactUnsupported(err) { + return ocispec.Descriptor{}, err + } + + if err := display.PrintStatus(root, "Fallback ", verbose); err != nil { + return ocispec.Descriptor{}, err + } + dst.SetReferrersCapability(false) + packOpts.PackImageManifest = true + root, err = pack() if err != nil { return ocispec.Descriptor{}, err } - if err = store.Tag(ctx, manifestDesc, tagStaged); err != nil { + copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if content.Equal(node, root) { + // skip non-config + content, err := content.FetchAll(ctx, fetcher, root) + if err != nil { + return nil, err + } + var manifest ocispec.Manifest + if err := json.Unmarshal(content, &manifest); err != nil { + return nil, err + } + return []ocispec.Descriptor{manifest.Config}, nil + } + + // config has no successors + return nil, nil + } + if err = copy(root); err != nil { return ocispec.Descriptor{}, err } - return manifestDesc, nil + return root, nil +} + +func isArtifactUnsupported(err error) bool { + var errResp *errcode.ErrorResponse + if !errors.As(err, &errResp) || errResp.StatusCode != http.StatusBadRequest { + return false + } + + var errCode errcode.Error + if !errors.As(errResp, &errCode) { + return false + } + + // As of November 2022, ECR is known to return UNSUPPORTED error when + // putting an OCI artifact manifest. + switch errCode.Code { + case errcode.ErrorCodeManifestInvalid, errcode.ErrorCodeUnsupported: + return true + } + return false } diff --git a/go.mod b/go.mod index ca7751cff..d616a0ce7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - oras.land/oras-go/v2 v2.0.0-rc.3.0.20221018111647-1969551cc3c7 + oras.land/oras-go/v2 v2.0.0-rc.4 ) require ( diff --git a/go.sum b/go.sum index d8f07140b..3d9b48f41 100644 --- a/go.sum +++ b/go.sum @@ -63,5 +63,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -oras.land/oras-go/v2 v2.0.0-rc.3.0.20221018111647-1969551cc3c7 h1:5ikyoiiYKWxQCxitZ+ZQ6KdxknZj1MdG+CX3iPGBrvw= -oras.land/oras-go/v2 v2.0.0-rc.3.0.20221018111647-1969551cc3c7/go.mod h1:YGHvWBGuqRlZgUyXUIoKsR3lcuCOb3DAtG0SEsEw1iY= +oras.land/oras-go/v2 v2.0.0-rc.4 h1:hg/R2znUQ1+qd43gRmL16VeX1GIZ8hQlLalBjYhhKSk= +oras.land/oras-go/v2 v2.0.0-rc.4/go.mod h1:YGHvWBGuqRlZgUyXUIoKsR3lcuCOb3DAtG0SEsEw1iY=