From 2cce40ecb23a993cceca9da9232d64bdb5f66987 Mon Sep 17 00:00:00 2001 From: Christian Weichel Date: Fri, 24 Sep 2021 15:46:56 +0000 Subject: [PATCH] [image-builder] Copy image rather than use buildkit To work around: https://github.com/containerd/containerd/issues/5978 --- .../image-builder-bob/pkg/builder/builder.go | 244 ++++++++++++++---- 1 file changed, 199 insertions(+), 45 deletions(-) diff --git a/components/image-builder-bob/pkg/builder/builder.go b/components/image-builder-bob/pkg/builder/builder.go index 09ab0f04cbeb8e..575f90e8a219cc 100644 --- a/components/image-builder-bob/pkg/builder/builder.go +++ b/components/image-builder-bob/pkg/builder/builder.go @@ -6,7 +6,10 @@ package builder import ( "context" + "encoding/json" "errors" + "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -17,14 +20,13 @@ import ( "github.com/gitpod-io/gitpod/common-go/log" "github.com/containerd/console" + "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/moby/buildkit/client" - "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/util/contentutil" - "github.com/moby/buildkit/util/imageutil" "github.com/moby/buildkit/util/progress/progressui" + "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" "golang.org/x/xerrors" @@ -193,20 +195,15 @@ func (b *Builder) buildBaseLayer(ctx context.Context, cl *client.Client) error { } func (b *Builder) buildWorkspaceImage(ctx context.Context, cl *client.Client) (err error) { - // Note: buildkit does not handle/export image config by default. That's why we need - // to download it ourselves and explicitely export it. - // See https://github.com/moby/buildkit/issues/2362 for details. - - var ( - sess []session.Attachable - resolver remotes.Resolver - ) + // Workaround: buildkit/containerd currently does not support pushing multi-image builds + // with some registries, e.g. gcr.io. Until https://github.com/containerd/containerd/issues/5978 + // is resolved, we'll manually copy the image. + var resolver remotes.Resolver if gplayerAuth := b.Config.WorkspaceLayerAuth; gplayerAuth != "" { - auth, err := newAuthProviderFromEnvvar(gplayerAuth) + _, err := newAuthProviderFromEnvvar(gplayerAuth) if err != nil { return err } - sess = append(sess, auth) authorizer, err := newDockerAuthorizerFromEnvvar(gplayerAuth) if err != nil { @@ -218,55 +215,212 @@ func (b *Builder) buildWorkspaceImage(ctx context.Context, cl *client.Client) (e } else { resolver = docker.NewResolver(docker.ResolverOptions{}) } + return copyImage(ctx, resolver, b.Config.BaseRef, b.Config.TargetRef) + + // // Note: buildkit does not handle/export image config by default. That's why we need + // // to download it ourselves and explicitely export it. + // // See https://github.com/moby/buildkit/issues/2362 for details. + // var sess []session.Attachable + // if gplayerAuth := b.Config.WorkspaceLayerAuth; gplayerAuth != "" { + // auth, err := newAuthProviderFromEnvvar(gplayerAuth) + // if err != nil { + // return err + // } + // sess = append(sess, auth) + + // authorizer, err := newDockerAuthorizerFromEnvvar(gplayerAuth) + // if err != nil { + // return err + // } + // resolver = docker.NewResolver(docker.ResolverOptions{ + // Authorizer: authorizer, + // }) + // } else { + // resolver = docker.NewResolver(docker.ResolverOptions{}) + // } + + // platform := specs.Platform{OS: "linux", Architecture: "amd64"} + // _, cfg, err := imageutil.Config(ctx, b.Config.BaseRef, resolver, contentutil.NewBuffer(), nil, &platform) + // if err != nil { + // return err + // } + // state, err := llb.Image(b.Config.BaseRef).WithImageConfig(cfg) + // if err != nil { + // return err + // } + + // def, err := state.Marshal(ctx, llb.Platform(platform)) + // if err != nil { + // return err + // } + + // // TODO(cw): + // // buildkit does not support setting raw annotations yet (https://github.com/moby/buildkit/issues/1220). + // // Once it does, we should set org.opencontainers.image.base.name as defined in https://github.com/opencontainers/image-spec/blob/main/annotations.md + + // solveOpt := client.SolveOpt{ + // Exports: []client.ExportEntry{ + // { + // Type: "image", + // Attrs: map[string]string{ + // "name": b.Config.TargetRef, + // "push": "true", + // "containerimage.config": string(cfg), + // }, + // }, + // }, + // Session: sess, + // CacheImports: b.Config.LocalCacheImport(), + // } + + // eg, ctx := errgroup.WithContext(ctx) + // ch := make(chan *client.SolveStatus) + // eg.Go(func() error { + // _, err := cl.Solve(ctx, def, solveOpt, ch) + // if err != nil { + // return xerrors.Errorf("cannot build Gitpod layer: %w", err) + // } + // return nil + // }) + // eg.Go(func() error { + // var c console.Console + // return progressui.DisplaySolveStatus(ctx, "", c, os.Stdout, ch) + // }) + // return eg.Wait() +} - platform := specs.Platform{OS: "linux", Architecture: "amd64"} - _, cfg, err := imageutil.Config(ctx, b.Config.BaseRef, resolver, contentutil.NewBuffer(), nil, &platform) +func copyImage(ctx context.Context, resolver remotes.Resolver, from, to string) error { + fromRef, fromDesc, err := resolver.Resolve(ctx, from) if err != nil { return err } - state, err := llb.Image(b.Config.BaseRef).WithImageConfig(cfg) + fetcher, err := resolver.Fetcher(ctx, fromRef) if err != nil { return err } - def, err := state.Marshal(ctx, llb.Platform(platform)) + fetch := func(desc specs.Descriptor, out interface{}) error { + rc, err := fetcher.Fetch(ctx, fromDesc) + if err != nil { + return err + } + defer rc.Close() + + return json.NewDecoder(rc).Decode(out) + } + if fromDesc.MediaType == specs.MediaTypeImageIndex { + var idx specs.Index + err := fetch(fromDesc, &idx) + if err != nil { + return err + } + + var res *specs.Descriptor + for _, m := range idx.Manifests { + if m.Platform != nil && m.Platform.Architecture == "amd64" && m.Platform.OS == "linux" { + res = &m + break + } + } + if res == nil { + return fmt.Errorf("no manifest for amd64/linux found") + } + fromDesc = *res + } + + var manifest specs.Manifest + err = fetch(fromDesc, &manifest) + if err != nil { + return err + } + manifestB, err := json.Marshal(manifest) if err != nil { return err } - // TODO(cw): - // buildkit does not support setting raw annotations yet (https://github.com/moby/buildkit/issues/1220). - // Once it does, we should set org.opencontainers.image.base.name as defined in https://github.com/opencontainers/image-spec/blob/main/annotations.md + pusher, err := resolver.Pusher(ctx, to) + if err != nil { + return err + } - solveOpt := client.SolveOpt{ - Exports: []client.ExportEntry{ - { - Type: "image", - Attrs: map[string]string{ - "name": b.Config.TargetRef, - "push": "true", - "containerimage.config": string(cfg), - }, - }, - }, - Session: sess, - CacheImports: b.Config.LocalCacheImport(), + // TODO(cw): optimize layer copy - copy only when pushing to a different registry + eg, egctx := errgroup.WithContext(ctx) + for _, layer := range manifest.Layers { + layer := layer + eg.Go(func() error { return copyLayer(egctx, fetcher, pusher, layer) }) + } + eg.Go(func() error { return copyLayer(egctx, fetcher, pusher, manifest.Config) }) + err = eg.Wait() + if err != nil { + return err } - eg, ctx := errgroup.WithContext(ctx) - ch := make(chan *client.SolveStatus) - eg.Go(func() error { - _, err := cl.Solve(ctx, def, solveOpt, ch) + mfspec := specs.Descriptor{ + MediaType: specs.MediaTypeImageManifest, + Digest: digest.FromBytes(manifestB), + Size: int64(len(manifestB)), + Platform: &specs.Platform{OS: "linux", Architecture: "amd64"}, + } + mfw, err := pusher.Push(ctx, mfspec) + if err != nil { + return err + } + defer mfw.Close() + mfwN, err := mfw.Write(manifestB) + if err != nil { + return err + } + if mfwN != len(manifestB) { + return fmt.Errorf("cannot write manifest: %w", io.ErrShortWrite) + } + err = mfw.Commit(ctx, mfspec.Size, mfspec.Digest) + if err != nil { + return err + } + + return nil +} + +func copyLayer(ctx context.Context, fetcher remotes.Fetcher, pusher remotes.Pusher, layer specs.Descriptor) (err error) { + log := log.WithField("blob", layer) + defer func() { + if errdefs.IsAlreadyExists(err) { + err = nil + } if err != nil { - return xerrors.Errorf("cannot build Gitpod layer: %w", err) + log.WithError(err).Error("failed to copy blob") + } else { + log.Info("push complete") } - return nil - }) - eg.Go(func() error { - var c console.Console - return progressui.DisplaySolveStatus(ctx, "", c, os.Stdout, ch) - }) - return eg.Wait() + }() + log.Info("pushing blob") + + in, err := fetcher.Fetch(ctx, layer) + if err != nil { + return err + } + defer in.Close() + + out, err := pusher.Push(ctx, layer) + if err != nil { + return err + } + defer out.Close() + + n, err := io.Copy(out, in) + if err != nil { + return err + } + if n != layer.Size { + return fmt.Errorf("copied less than %d bytes from layer %s (expected %d bytes)", n, layer.Digest.String(), layer.Size) + } + + err = out.Commit(ctx, n, layer.Digest) + if err != nil { + return err + } + + return nil } func waitForBuildContext(ctx context.Context) error {