Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Imagetools multiple repositories #1137

Merged
merged 4 commits into from
Aug 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -925,7 +925,21 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s

itpull := imagetools.New(imageopt)

dt, desc, err := itpull.Combine(ctx, names[0], descs)
ref, err := reference.ParseNormalizedNamed(names[0])
if err != nil {
return err
}
ref = reference.TagNameOnly(ref)

srcs := make([]*imagetools.Source, len(descs))
for i, desc := range descs {
srcs[i] = &imagetools.Source{
Desc: desc,
Ref: ref,
}
}

dt, desc, err := itpull.Combine(ctx, srcs)
if err != nil {
return err
}
Expand Down
85 changes: 56 additions & 29 deletions commands/imagetools/create.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package commands

import (
"context"
"encoding/json"
"fmt"
"os"
Expand All @@ -9,6 +10,7 @@ import (
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/appcontext"
Expand All @@ -25,6 +27,7 @@ type createOptions struct {
tags []string
dryrun bool
actionAppend bool
progress string
}

func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
Expand Down Expand Up @@ -78,18 +81,21 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
if len(repos) == 0 {
return errors.Errorf("no repositories specified, please set a reference in tag or source")
}
if len(repos) > 1 {
return errors.Errorf("multiple repositories currently not supported, found %v", repos)
}

var repo string
for r := range repos {
repo = r
var defaultRepo *string
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The repos map looks unused now. You can just use the earlier range tags/srcs to determine the defaultRepo

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

repos is only used to determine the defaultRepo. I think some sort of collection of all the repo names is required, since we only derive a default if all the other repo names present are the same.

We could potentially have a bool to track if we found a different repo that doesn't match the previously selected default during iteration, but I think that's potentially more complex than just tracking all the found repo names and then having a len().

if len(repos) == 1 {
for repo := range repos {
defaultRepo = &repo
}
}

for i, s := range srcs {
if s.Ref == nil && s.Desc.MediaType == "" && s.Desc.Digest != "" {
n, err := reference.ParseNormalizedNamed(repo)
if defaultRepo == nil {
return errors.Errorf("multiple repositories specified, cannot infer repository for %q", args[i])
}

n, err := reference.ParseNormalizedNamed(*defaultRepo)
if err != nil {
return err
}
Expand Down Expand Up @@ -143,7 +149,6 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
if err != nil {
return err
}
srcs[i].Ref = nil
if srcs[i].Desc.Digest == "" {
srcs[i].Desc = desc
} else {
Expand All @@ -162,12 +167,7 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
}
}

descs := make([]ocispec.Descriptor, len(srcs))
for i := range descs {
descs[i] = srcs[i].Desc
}

dt, desc, err := r.Combine(ctx, repo, descs)
dt, desc, err := r.Combine(ctx, srcs)
if err != nil {
return err
}
Expand All @@ -180,23 +180,49 @@ func runCreate(dockerCli command.Cli, in createOptions, args []string) error {
// new resolver cause need new auth
r = imagetools.New(imageopt)

ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
printer := progress.NewPrinter(ctx2, os.Stderr, os.Stderr, in.progress)

eg, _ := errgroup.WithContext(ctx)
pw := progress.WithPrefix(printer, "internal", true)

for _, t := range tags {
if err := r.Push(ctx, t, desc, dt); err != nil {
return err
}
fmt.Println(t.String())
t := t
eg.Go(func() error {
return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
eg2, _ := errgroup.WithContext(ctx)
for _, s := range srcs {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this only happen when src and destination repo are actually different?

if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
continue
}
s := s
eg2.Go(func() error {
sub.Log(1, []byte(fmt.Sprintf("copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String())))
return r.Copy(ctx, s, t)
})
}

if err := eg2.Wait(); err != nil {
return err
}
sub.Log(1, []byte(fmt.Sprintf("pushing %s to %s\n", desc.Digest.String(), t.String())))
return r.Push(ctx, t, desc, dt)
})
})
}

return nil
}
err = eg.Wait()
err1 := printer.Wait()
if err == nil {
err = err1
}

type src struct {
Desc ocispec.Descriptor
Ref reference.Named
return err
}

func parseSources(in []string) ([]*src, error) {
out := make([]*src, len(in))
func parseSources(in []string) ([]*imagetools.Source, error) {
out := make([]*imagetools.Source, len(in))
for i, in := range in {
s, err := parseSource(in)
if err != nil {
Expand All @@ -219,11 +245,11 @@ func parseRefs(in []string) ([]reference.Named, error) {
return refs, nil
}

func parseSource(in string) (*src, error) {
func parseSource(in string) (*imagetools.Source, error) {
// source can be a digest, reference or a descriptor JSON
dgst, err := digest.Parse(in)
if err == nil {
return &src{
return &imagetools.Source{
Desc: ocispec.Descriptor{
Digest: dgst,
},
Expand All @@ -234,14 +260,14 @@ func parseSource(in string) (*src, error) {

ref, err := reference.ParseNormalizedNamed(in)
if err == nil {
return &src{
return &imagetools.Source{
Ref: ref,
}, nil
} else if !strings.HasPrefix(in, "{") {
return nil, err
}

var s src
var s imagetools.Source
if err := json.Unmarshal([]byte(in), &s.Desc); err != nil {
return nil, errors.WithStack(err)
}
Expand All @@ -265,6 +291,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
flags.StringArrayVarP(&options.tags, "tag", "t", []string{}, "Set reference for new image")
flags.BoolVar(&options.dryrun, "dry-run", false, "Show final image instead of pushing")
flags.BoolVar(&options.actionAppend, "append", false, "Append to existing manifest")
flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty"). Use plain to show container output`)

return cmd
}
Expand Down
1 change: 1 addition & 0 deletions docs/reference/buildx_imagetools_create.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Create a new image based on source images
| [`--builder`](#builder) | `string` | | Override the configured builder instance |
| [`--dry-run`](#dry-run) | | | Show final image instead of pushing |
| [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file |
| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`). Use plain to show container output |
| [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image |


Expand Down
76 changes: 55 additions & 21 deletions util/imagetools/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,59 +4,62 @@ import (
"bytes"
"context"
"encoding/json"
"net/url"
"strings"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/docker/distribution/reference"
"github.com/moby/buildkit/util/contentutil"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)

func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) {
ref, err := parseRef(in)
if err != nil {
return nil, ocispec.Descriptor{}, err
}
type Source struct {
Desc ocispec.Descriptor
Ref reference.Named
}

func (r *Resolver) Combine(ctx context.Context, srcs []*Source) ([]byte, ocispec.Descriptor, error) {
eg, ctx := errgroup.WithContext(ctx)

dts := make([][]byte, len(descs))
dts := make([][]byte, len(srcs))
for i := range dts {
func(i int) {
eg.Go(func() error {
dt, err := r.GetDescriptor(ctx, ref.String(), descs[i])
dt, err := r.GetDescriptor(ctx, srcs[i].Ref.String(), srcs[i].Desc)
if err != nil {
return err
}
dts[i] = dt

if descs[i].MediaType == "" {
if srcs[i].Desc.MediaType == "" {
mt, err := detectMediaType(dt)
if err != nil {
return err
}
descs[i].MediaType = mt
srcs[i].Desc.MediaType = mt
}

mt := descs[i].MediaType
mt := srcs[i].Desc.MediaType

switch mt {
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p := descs[i].Platform
if descs[i].Platform == nil {
p := srcs[i].Desc.Platform
if srcs[i].Desc.Platform == nil {
p = &ocispec.Platform{}
}
if p.OS == "" || p.Architecture == "" {
if err := r.loadPlatform(ctx, p, in, dt); err != nil {
if err := r.loadPlatform(ctx, p, srcs[i].Ref.String(), dt); err != nil {
return err
}
}
descs[i].Platform = p
srcs[i].Desc.Platform = p
case images.MediaTypeDockerSchema1Manifest:
return errors.Errorf("schema1 manifests are not allowed in manifest lists")
}
Expand All @@ -71,14 +74,14 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
}

// on single source, return original bytes
if len(descs) == 1 {
if mt := descs[0].MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
return dts[0], descs[0], nil
if len(srcs) == 1 {
if mt := srcs[0].Desc.MediaType; mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex {
return dts[0], srcs[0].Desc, nil
}
}

m := map[digest.Digest]int{}
newDescs := make([]ocispec.Descriptor, 0, len(descs))
newDescs := make([]ocispec.Descriptor, 0, len(srcs))

addDesc := func(d ocispec.Descriptor) {
idx, ok := m[d.Digest]
Expand All @@ -103,8 +106,8 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
}
}

for i, desc := range descs {
switch desc.MediaType {
for i, src := range srcs {
switch src.Desc.MediaType {
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
var mfst ocispec.Index
if err := json.Unmarshal(dts[i], &mfst); err != nil {
Expand All @@ -114,7 +117,7 @@ func (r *Resolver) Combine(ctx context.Context, in string, descs []ocispec.Descr
addDesc(d)
}
default:
addDesc(desc)
addDesc(src.Desc)
}
}

Expand Down Expand Up @@ -169,6 +172,37 @@ func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispec.D
return err
}

func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) error {
dest = reference.TagNameOnly(dest)
p, err := r.resolver().Pusher(ctx, dest.String())
if err != nil {
return err
}

srcRef := reference.TagNameOnly(src.Ref)
f, err := r.resolver().Fetcher(ctx, srcRef.String())
if err != nil {
return err
}

refspec := reference.TrimNamed(src.Ref).String()
u, err := url.Parse("dummy://" + refspec)
if err != nil {
return err
}
source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/")
if src.Desc.Annotations == nil {
src.Desc.Annotations = make(map[string]string)
}
src.Desc.Annotations["containerd.io/distribution.source."+source] = repo

err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), src.Desc)
if err != nil {
return err
}
return nil
}

func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispec.Platform, in string, dt []byte) error {
var manifest ocispec.Manifest
if err := json.Unmarshal(dt, &manifest); err != nil {
Expand Down