Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enhance manifest fetch #541

Merged
merged 16 commits into from
Sep 16, 2022
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/oras/manifest/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

func Cmd() *cobra.Command {
cmd := &cobra.Command{
Use: "manifest [fetch]",
Use: "manifest [command]",
Short: "[Preview] Manifest operations",
}

Expand Down
115 changes: 86 additions & 29 deletions cmd/oras/manifest/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,30 @@ limitations under the License.
package manifest

import (
"bytes"
"encoding/json"
"fmt"
"errors"
"os"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/oci"
oerrors "oras.land/oras/cmd/oras/internal/errors"
"oras.land/oras/cmd/oras/internal/option"
"oras.land/oras/internal/cas"
"oras.land/oras/internal/cache"
)

type fetchOptions struct {
option.Common
option.Descriptor
option.Remote
option.Platform
option.Pretty

targetRef string
pretty bool
mediaTypes []string
fetchDescriptor bool
cacheRoot string
mediaTypes []string
outputPath string
targetRef string
}

func fetchCmd() *cobra.Command {
Expand All @@ -44,6 +48,7 @@ func fetchCmd() *cobra.Command {
Use: "fetch [flags] <name:tag|name@digest>",
Short: "[Preview] Fetch manifest of the target artifact",
Long: `[Preview] Fetch manifest of the target artifact

** This command is in preview and under development. **

Example - Fetch raw manifest:
Expand All @@ -63,6 +68,11 @@ Example - Fetch manifest with prettified json result:
`,
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
if opts.outputPath == "-" && opts.OutputDescriptor {
return errors.New("`--output -` cannot be used with `--descriptor` at the same time")
}

opts.cacheRoot = os.Getenv("ORAS_CACHE")
return opts.ReadPassword()
},
Aliases: []string{"get"},
Expand All @@ -72,46 +82,93 @@ Example - Fetch manifest with prettified json result:
},
}

cmd.Flags().BoolVarP(&opts.pretty, "pretty", "", false, "output prettified manifest")
cmd.Flags().BoolVarP(&opts.fetchDescriptor, "descriptor", "", false, "fetch a descriptor of the manifest")
cmd.Flags().StringSliceVarP(&opts.mediaTypes, "media-type", "", nil, "accepted media types")
cmd.Flags().StringVarP(&opts.outputPath, "output", "o", "", "output file path")
option.ApplyFlags(&opts, cmd.Flags())
return cmd
}

func fetchManifest(opts fetchOptions) error {
func fetchManifest(opts fetchOptions) (fetchErr error) {
ctx, _ := opts.SetLoggerLevel()
targetPlatform, err := opts.Parse()
if err != nil {
return err
}

repo, err := opts.NewRepository(opts.targetRef, opts.Common)
if err != nil {
return err
}

if repo.Reference.Reference == "" {
return oerrors.NewErrInvalidReference(repo.Reference)
}
repo.ManifestMediaTypes = opts.mediaTypes

// Fetch and output
var content []byte
if opts.fetchDescriptor {
content, err = cas.FetchDescriptor(ctx, repo, opts.targetRef, targetPlatform)
} else {
content, err = cas.FetchManifest(ctx, repo, opts.targetRef, targetPlatform)
}
targetPlatform, err := opts.Parse()
if err != nil {
return err
}
if opts.pretty {
buf := bytes.NewBuffer(nil)
if err = json.Indent(buf, content, "", " "); err != nil {
return fmt.Errorf("failed to prettify: %w", err)

var src oras.ReadOnlyTarget = repo.Manifests()
if opts.cacheRoot != "" {
ociStore, err := oci.New(opts.cacheRoot)
if err != nil {
return err
}
buf.WriteByte('\n')
content = buf.Bytes()
src = cache.New(src, ociStore)
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we refactor it into the option package?

/cc @qweeah

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we refactor it into the option package?

/cc @qweeah

Created a option Cache.

_, err = os.Stdout.Write(content)
return err

var desc ocispec.Descriptor
var content []byte
yuehaoliang marked this conversation as resolved.
Show resolved Hide resolved
if opts.OutputDescriptor && opts.outputPath == "" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This if block can be merged with the if block at the end:

  1. If !opts.OutputDescriptor || opt.outputPath != "" => fetch manifest content
  2. If opts.OutputDescriptor => resolve and output to stdout

Copy link
Contributor Author

@yuehaoliang yuehaoliang Sep 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't really follow your idea here.

In the case of fetching manifest content, there still might be a need to output descriptor, eg. --output manifest.json --descriptor. Since oras.FetchBytes returns both the content and the descriptor, so if content has already been fetched, we don't need to call oras.Resolve again.

Current logic:

if opts.OutputDescriptor && opts.outputPath == "" {
	// oras.Resolve   fetch manifest descriptor only
} else {
	// oras.FetchBytes   fetch manifest descriptor content

	if opts.outputPath == "" || opts.outputPath == "-" {
		// output manifest content

		return
	}

	// save manifest content into the local file
}

if opts.OutputDescriptor {
	// output manifest descriptor
}

In the logic below, for --output manifest.json --descriptor, there's a redundant call of oras.Resolve.

if !opts.OutputDescriptor || opt.outputPath != "" {
	// oras.FetchBytes   fetch manifest descriptor content

	if opts.outputPath == "" || opts.outputPath == "-" {
		// output manifest content

		return
	}

	// save manifest content into the local file
} 

if opts.OutputDescriptor {
	// oras.Resolve   fetch manifest descriptor only

	// output manifest descriptor
}

Copy link
Contributor

@qweeah qweeah Sep 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since oras.FetchBytes returns both the content and the descriptor, so if content has already been fetched, we don't need to call oras.Resolve again

This can be avoided by checking if the desc is an empty struct.

// fetch manifest descriptor only
desc, err = oras.Resolve(ctx, src, opts.targetRef, oras.DefaultResolveOptions)
if err != nil {
return err
}
} else {
// fetch manifest content
desc, content, err = oras.FetchBytes(ctx, src, opts.targetRef, oras.FetchBytesOptions{
FetchOptions: oras.FetchOptions{
ResolveOptions: oras.ResolveOptions{
TargetPlatform: targetPlatform,
},
},
MaxBytes: 0,
})
yuehaoliang marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return err
}

// outputs manifest content
yuehaoliang marked this conversation as resolved.
Show resolved Hide resolved
if opts.outputPath == "" || opts.outputPath == "-" {
return opts.Output(os.Stdout, content)
}

// save manifest content into the local file if the output path is provided
file, err := os.Create(opts.outputPath)
if err != nil {
return err
}
defer func() {
if err := file.Close(); fetchErr == nil {
fetchErr = err
}
}()

if _, err = file.Write(content); err != nil {
return err
}
yuehaoliang marked this conversation as resolved.
Show resolved Hide resolved
}

// output manifest's descriptor if `--descriptor` is used
if opts.OutputDescriptor {
descBytes, err := json.Marshal(desc)
if err != nil {
return err
}
err = opts.Output(os.Stdout, descBytes)
if err != nil {
return err
}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code block appears everywhere if --descriptor and --pretty and both needed, any idea to refactor it? It doesn't necessarily need to be done in this PR /cc @lizMSFT

Copy link
Contributor Author

@yuehaoliang yuehaoliang Sep 15, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're discussing about this change offline.


return nil
}
71 changes: 0 additions & 71 deletions internal/cas/fetch.go

This file was deleted.

Loading