Skip to content

Commit

Permalink
feat!: support OCI image fallback for oras push and oras attach (o…
Browse files Browse the repository at this point in the history 
…ras-project#665)

Since oras-go will take care of fallback with tag schema if referrer API
not supported, this PR provides OCI Image fallback when-and-only-when
OCI artifact is not supported.

Resolves: oras-project#654

Signed-off-by: Billy Zha <[email protected]>
  • Loading branch information
@qweeah
qweeah authored and Terry Howe committed Feb 2, 2023
1 parent 638e31f commit b7d9cfb
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 86 deletions.
55 changes: 20 additions & 35 deletions cmd/oras/attach.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ import (
"context"
"errors"
"fmt"
"sync"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras/cmd/oras/internal/display"
oerrors "oras.land/oras/cmd/oras/internal/errors"
"oras.land/oras/cmd/oras/internal/option"
)
Expand Down Expand Up @@ -113,42 +111,33 @@ func runAttach(opts attachOptions) error {
if err != nil {
return err
}
root, err := oras.Pack(
ctx, store, opts.artifactType, descs,
oras.PackOptions{
Subject: &subject,
ManifestAnnotations: annotations[option.AnnotationManifest],
})
if err != nil {
return err
}

// prepare push
committed := &sync.Map{}
graphCopyOptions := oras.DefaultCopyGraphOptions
graphCopyOptions.Concurrency = opts.concurrency
graphCopyOptions.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if isEqualOCIDescriptor(node, root) {
// skip subject
return descs, nil
}
return content.Successors(ctx, fetcher, node)
packOpts := oras.PackOptions{
Subject: &subject,
ManifestAnnotations: annotations[option.AnnotationManifest],
}
graphCopyOptions.PreCopy = display.StatusPrinter("Uploading", opts.Verbose)
graphCopyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
return display.PrintStatus(desc, "Exists ", opts.Verbose)
pack := func() (ocispec.Descriptor, error) {
return oras.Pack(ctx, store, opts.artifactType, descs, packOpts)
}
graphCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, opts.Verbose); err != nil {
return err

graphCopyOptions := oras.DefaultCopyGraphOptions
graphCopyOptions.Concurrency = opts.concurrency
updateDisplayOption(&graphCopyOptions, store, opts.Verbose)
copy := func(root ocispec.Descriptor) error {
if root.MediaType == ocispec.MediaTypeArtifactManifest {
graphCopyOptions.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if content.Equal(node, root) {
// skip subject
return descs, nil
}
return content.Successors(ctx, fetcher, node)
}
}
return display.PrintStatus(desc, "Uploaded ", opts.Verbose)
return oras.CopyGraph(ctx, store, dst, root, graphCopyOptions)
}

// push
err = oras.CopyGraph(ctx, store, dst, root, graphCopyOptions)
root, err := pushArtifact(dst, pack, &packOpts, copy, &graphCopyOptions, opts.Verbose)
if err != nil {
return err
}
Expand All @@ -159,7 +148,3 @@ func runAttach(opts attachOptions) error {
// Export manifest
return opts.ExportManifest(ctx, store, root)
}

func isEqualOCIDescriptor(a, b ocispec.Descriptor) bool {
return a.Size == b.Size && a.Digest == b.Digest && a.MediaType == b.MediaType
}
180 changes: 132 additions & 48 deletions cmd/oras/push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ package main

import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"

Expand All @@ -27,14 +29,12 @@ import (
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/errcode"
"oras.land/oras/cmd/oras/internal/display"
"oras.land/oras/cmd/oras/internal/option"
)

const (
tagStaged = "staged"
)

type pushOptions struct {
option.Common
option.Remote
Expand Down Expand Up @@ -120,94 +120,178 @@ func runPush(opts pushOptions) error {
return err
}

// Prepare manifest
// prepare pack
packOpts := oras.PackOptions{
ConfigAnnotations: annotations[option.AnnotationConfig],
ManifestAnnotations: annotations[option.AnnotationManifest],
}
store := file.New("")
defer store.Close()
store.AllowPathTraversalOnWrite = opts.PathValidationDisabled

// Ready to push
committed := &sync.Map{}
copyOptions := oras.DefaultCopyOptions
copyOptions.Concurrency = opts.concurrency
copyOptions.PreCopy = display.StatusPrinter("Uploading", opts.Verbose)
copyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
return display.PrintStatus(desc, "Exists ", opts.Verbose)
}
copyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, opts.Verbose); err != nil {
if opts.manifestConfigRef != "" {
path, cfgMediaType := parseFileReference(opts.manifestConfigRef, oras.MediaTypeUnknownConfig)
desc, err := store.Add(ctx, option.AnnotationConfig, cfgMediaType, path)
if err != nil {
return err
}
return display.PrintStatus(desc, "Uploaded ", opts.Verbose)
desc.Annotations = packOpts.ConfigAnnotations
packOpts.ConfigDescriptor = &desc
packOpts.PackImageManifest = true
}
desc, err := packManifest(ctx, store, annotations, &opts)
descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose)
if err != nil {
return err
}
pack := func() (ocispec.Descriptor, error) {
root, err := oras.Pack(ctx, store, opts.artifactType, descs, packOpts)
if err != nil {
return ocispec.Descriptor{}, err
}
if err = store.Tag(ctx, root, root.Digest.String()); err != nil {
return ocispec.Descriptor{}, err
}
return root, nil
}

// Push
// prepare push
dst, err := opts.NewRepository(opts.targetRef, opts.Common)
if err != nil {
return err
}
if tag := dst.Reference.Reference; tag == "" {
err = oras.CopyGraph(ctx, store, dst, desc, copyOptions.CopyGraphOptions)
} else {
desc, err = oras.Copy(ctx, store, tagStaged, dst, tag, copyOptions)
copyOptions := oras.DefaultCopyOptions
copyOptions.Concurrency = opts.concurrency
updateDisplayOption(&copyOptions.CopyGraphOptions, store, opts.Verbose)
copy := func(root ocispec.Descriptor) error {
if tag := dst.Reference.Reference; tag == "" {
err = oras.CopyGraph(ctx, store, dst, root, copyOptions.CopyGraphOptions)
} else {
_, err = oras.Copy(ctx, store, root.Digest.String(), dst, tag, copyOptions)
}
return err
}

// Push
root, err := pushArtifact(dst, pack, &packOpts, copy, &copyOptions.CopyGraphOptions, opts.Verbose)
if err != nil {
return err
}

fmt.Println("Pushed", opts.targetRef)

if len(opts.extraRefs) != 0 {
contentBytes, err := content.FetchAll(ctx, store, desc)
contentBytes, err := content.FetchAll(ctx, store, root)
if err != nil {
return err
}
tagBytesNOpts := oras.DefaultTagBytesNOptions
tagBytesNOpts.Concurrency = opts.concurrency
if _, err = oras.TagBytesN(ctx, &display.TagManifestStatusPrinter{Repository: dst}, desc.MediaType, contentBytes, opts.extraRefs, tagBytesNOpts); err != nil {
if _, err = oras.TagBytesN(ctx, &display.TagManifestStatusPrinter{Repository: dst}, root.MediaType, contentBytes, opts.extraRefs, tagBytesNOpts); err != nil {
return err
}
}

fmt.Println("Digest:", desc.Digest)
fmt.Println("Digest:", root.Digest)

// Export manifest
return opts.ExportManifest(ctx, store, desc)
return opts.ExportManifest(ctx, store, root)
}

func packManifest(ctx context.Context, store *file.Store, annotations map[string]map[string]string, opts *pushOptions) (ocispec.Descriptor, error) {
var packOpts oras.PackOptions
packOpts.ConfigAnnotations = annotations[option.AnnotationConfig]
packOpts.ManifestAnnotations = annotations[option.AnnotationManifest]

if opts.manifestConfigRef != "" {
path, mediatype := parseFileReference(opts.manifestConfigRef, oras.MediaTypeUnknownConfig)
desc, err := store.Add(ctx, option.AnnotationConfig, mediatype, path)
if err != nil {
return ocispec.Descriptor{}, err
func updateDisplayOption(opts *oras.CopyGraphOptions, store content.Fetcher, verbose bool) {
committed := &sync.Map{}
opts.PreCopy = display.StatusPrinter("Uploading", verbose)
opts.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
return display.PrintStatus(desc, "Exists ", verbose)
}
opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
if err := display.PrintSuccessorStatus(ctx, desc, "Skipped ", store, committed, verbose); err != nil {
return err
}
desc.Annotations = packOpts.ConfigAnnotations
packOpts.ConfigDescriptor = &desc
packOpts.PackImageManifest = true
return display.PrintStatus(desc, "Uploaded ", verbose)
}
descs, err := loadFiles(ctx, store, annotations, opts.FileRefs, opts.Verbose)
}

type packFunc func() (ocispec.Descriptor, error)
type copyFunc func(desc ocispec.Descriptor) error

func pushArtifact(dst *remote.Repository, pack packFunc, packOpts *oras.PackOptions, copy copyFunc, copyOpts *oras.CopyGraphOptions, verbose bool) (ocispec.Descriptor, error) {
root, err := pack()
if err != nil {
return ocispec.Descriptor{}, err
}

// pack artifact
manifestDesc, err := oras.Pack(ctx, store, opts.artifactType, descs, packOpts)
copyRootAttempted := false
preCopy := copyOpts.PreCopy
copyOpts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
if content.Equal(root, desc) {
// copyRootAttempted helps track whether the returned error is
// generated from copying root.
copyRootAttempted = true
}
if preCopy != nil {
return preCopy(ctx, desc)
}
return nil
}

// push
if err = copy(root); err == nil {
return root, nil
}

if !copyRootAttempted || !isArtifactUnsupported(err) {
return ocispec.Descriptor{}, err
}

if err := display.PrintStatus(root, "Fallback ", verbose); err != nil {
return ocispec.Descriptor{}, err
}
dst.SetReferrersCapability(false)
packOpts.PackImageManifest = true
root, err = pack()
if err != nil {
return ocispec.Descriptor{}, err
}

if err = store.Tag(ctx, manifestDesc, tagStaged); err != nil {
copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
if content.Equal(node, root) {
// skip non-config
content, err := content.FetchAll(ctx, fetcher, root)
if err != nil {
return nil, err
}
var manifest ocispec.Manifest
if err := json.Unmarshal(content, &manifest); err != nil {
return nil, err
}
return []ocispec.Descriptor{manifest.Config}, nil
}

// config has no successors
return nil, nil
}
if err = copy(root); err != nil {
return ocispec.Descriptor{}, err
}
return manifestDesc, nil
return root, nil
}

func isArtifactUnsupported(err error) bool {
var errResp *errcode.ErrorResponse
if !errors.As(err, &errResp) || errResp.StatusCode != http.StatusBadRequest {
return false
}

var errCode errcode.Error
if !errors.As(errResp, &errCode) {
return false
}

// As of November 2022, ECR is known to return UNSUPPORTED error when
// putting an OCI artifact manifest.
switch errCode.Code {
case errcode.ErrorCodeManifestInvalid, errcode.ErrorCodeUnsupported:
return true
}
return false
}
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ require (
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
oras.land/oras-go/v2 v2.0.0-rc.3.0.20221018111647-1969551cc3c7
oras.land/oras-go/v2 v2.0.0-rc.4
)

require (
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,5 +63,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
oras.land/oras-go/v2 v2.0.0-rc.3.0.20221018111647-1969551cc3c7 h1:5ikyoiiYKWxQCxitZ+ZQ6KdxknZj1MdG+CX3iPGBrvw=
oras.land/oras-go/v2 v2.0.0-rc.3.0.20221018111647-1969551cc3c7/go.mod h1:YGHvWBGuqRlZgUyXUIoKsR3lcuCOb3DAtG0SEsEw1iY=
oras.land/oras-go/v2 v2.0.0-rc.4 h1:hg/R2znUQ1+qd43gRmL16VeX1GIZ8hQlLalBjYhhKSk=
oras.land/oras-go/v2 v2.0.0-rc.4/go.mod h1:YGHvWBGuqRlZgUyXUIoKsR3lcuCOb3DAtG0SEsEw1iY=

0 comments on commit b7d9cfb

Please sign in to comment.