From 6f48f5d5909df492546e8c3d3d1bcaa25f50e129 Mon Sep 17 00:00:00 2001 From: Yan Song Date: Thu, 20 Jul 2023 11:28:39 +0000 Subject: [PATCH] nydusify: introduce copy subcommand `nydusify copy` copies an image from source registry to target registry, it also supports to specify a backend storage. Signed-off-by: Yan Song --- contrib/nydusify/cmd/nydusify.go | 149 ++++++++- contrib/nydusify/pkg/backend/backend.go | 3 + contrib/nydusify/pkg/backend/oss.go | 20 ++ contrib/nydusify/pkg/backend/registry.go | 9 + contrib/nydusify/pkg/backend/s3.go | 9 + .../pkg/converter/provider/provider.go | 4 + contrib/nydusify/pkg/copier/copier.go | 292 ++++++++++++++++++ contrib/nydusify/pkg/copier/store.go | 45 +++ contrib/nydusify/pkg/packer/pusher_test.go | 9 + 9 files changed, 530 insertions(+), 10 deletions(-) create mode 100644 contrib/nydusify/pkg/copier/copier.go create mode 100644 contrib/nydusify/pkg/copier/store.go diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index ae13d737caf..a37eed989fb 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -25,6 +25,7 @@ import ( "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/copier" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" @@ -67,27 +68,27 @@ func parseBackendConfig(backendConfigJSON, backendConfigFile string) (string, er return backendConfigJSON, nil } -func getBackendConfig(c *cli.Context, required bool) (string, string, error) { - backendType := c.String("backend-type") +func getBackendConfig(c *cli.Context, suffix string, required bool) (string, string, error) { + backendType := c.String(suffix + "backend-type") if backendType == "" { if required { - return "", "", errors.Errorf("backend type is empty, please specify option '--backend-type'") + return "", "", errors.Errorf("backend type is empty, please specify option '--%sbackend-type'", suffix) } return "", "", nil } possibleBackendTypes := []string{"oss", "s3"} if !isPossibleValue(possibleBackendTypes, backendType) { - return "", "", fmt.Errorf("--backend-type should be one of %v", possibleBackendTypes) + return "", "", fmt.Errorf("--%sbackend-type should be one of %v", suffix, possibleBackendTypes) } backendConfig, err := parseBackendConfig( - c.String("backend-config"), c.String("backend-config-file"), + c.String(suffix+"backend-config"), c.String(suffix+"backend-config-file"), ) if err != nil { return "", "", err } else if (backendType == "oss" || backendType == "s3") && strings.TrimSpace(backendConfig) == "" { - return "", "", errors.Errorf("backend configuration is empty, please specify option '--backend-config'") + return "", "", errors.Errorf("backend configuration is empty, please specify option '--%sbackend-config'", suffix) } return backendType, backendConfig, nil @@ -427,7 +428,7 @@ func main() { return err } - backendType, backendConfig, err := getBackendConfig(c, false) + backendType, backendConfig, err := getBackendConfig(c, "", false) if err != nil { return err } @@ -602,7 +603,7 @@ func main() { Action: func(c *cli.Context) error { setupLogLevel(c) - backendType, backendConfig, err := getBackendConfig(c, false) + backendType, backendConfig, err := getBackendConfig(c, "", false) if err != nil { return err } @@ -699,7 +700,7 @@ func main() { Action: func(c *cli.Context) error { setupLogLevel(c) - backendType, backendConfig, err := getBackendConfig(c, false) + backendType, backendConfig, err := getBackendConfig(c, "", false) if err != nil { return err } else if backendConfig == "" { @@ -875,7 +876,7 @@ func main() { // if backend-push is specified, we should make sure backend-config-file exists if c.Bool("backend-push") || c.Bool("compact") { - _backendType, _backendConfig, err := getBackendConfig(c, true) + _backendType, _backendConfig, err := getBackendConfig(c, "", true) if err != nil { return err } @@ -915,6 +916,134 @@ func main() { return nil }, }, + { + Name: "copy", + Usage: "Copy an image from source to target", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "source", + Required: true, + Usage: "Source image reference", + EnvVars: []string{"SOURCE"}, + }, + &cli.StringFlag{ + Name: "target", + Required: false, + Usage: "Target image reference", + EnvVars: []string{"TARGET"}, + }, + &cli.BoolFlag{ + Name: "source-insecure", + Required: false, + Usage: "Skip verifying server certs for HTTPS source registry", + EnvVars: []string{"SOURCE_INSECURE"}, + }, + &cli.BoolFlag{ + Name: "target-insecure", + Required: false, + Usage: "Skip verifying server certs for HTTPS target registry", + EnvVars: []string{"TARGET_INSECURE"}, + }, + + &cli.StringFlag{ + Name: "source-backend-type", + Value: "", + Usage: "Type of storage backend, possible values: 'oss', 's3'", + EnvVars: []string{"BACKEND_TYPE"}, + }, + &cli.StringFlag{ + Name: "source-backend-config", + Value: "", + Usage: "Json configuration string for storage backend", + EnvVars: []string{"BACKEND_CONFIG"}, + }, + &cli.PathFlag{ + Name: "source-backend-config-file", + Value: "", + TakesFile: true, + Usage: "Json configuration file for storage backend", + EnvVars: []string{"BACKEND_CONFIG_FILE"}, + }, + + &cli.StringFlag{ + Name: "target-backend-type", + Value: "", + Usage: "Type of storage backend, possible values: 'oss', 's3'", + EnvVars: []string{"BACKEND_TYPE"}, + }, + &cli.StringFlag{ + Name: "target-backend-config", + Value: "", + Usage: "Json configuration string for storage backend", + EnvVars: []string{"BACKEND_CONFIG"}, + }, + &cli.PathFlag{ + Name: "target-backend-config-file", + Value: "", + TakesFile: true, + Usage: "Json configuration file for storage backend", + EnvVars: []string{"BACKEND_CONFIG_FILE"}, + }, + + &cli.BoolFlag{ + Name: "all-platforms", + Value: false, + Usage: "Convert images for all platforms, conflicts with --platform", + }, + &cli.StringFlag{ + Name: "platform", + Value: "linux/" + runtime.GOARCH, + Usage: "Convert images for specific platforms, for example: 'linux/amd64,linux/arm64'", + }, + + &cli.StringFlag{ + Name: "work-dir", + Value: "./tmp", + Usage: "Working directory for image conversion", + EnvVars: []string{"WORK_DIR"}, + }, + &cli.StringFlag{ + Name: "nydus-image", + Value: "nydus-image", + Usage: "Path to the nydus-image binary, default to search in PATH", + EnvVars: []string{"NYDUS_IMAGE"}, + }, + }, + Action: func(c *cli.Context) error { + setupLogLevel(c) + + sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false) + if err != nil { + return err + } + + targetBackendType, targetBackendConfig, err := getBackendConfig(c, "target-", false) + if err != nil { + return err + } + + opt := copier.Opt{ + WorkDir: c.String("work-dir"), + NydusImagePath: c.String("nydus-image"), + + Source: c.String("source"), + Target: c.String("target"), + SourceInsecure: c.Bool("source-insecure"), + TargetInsecure: c.Bool("target-insecure"), + + SourceBackendType: sourceBackendType, + SourceBackendConfig: sourceBackendConfig, + + TargetBackendType: targetBackendType, + TargetBackendConfig: targetBackendConfig, + + AllPlatforms: c.Bool("all-platforms"), + Platforms: c.String("platform"), + } + + return copier.Copy(context.Background(), opt) + }, + }, } if !utils.IsSupportedArch(runtime.GOARCH) { diff --git a/contrib/nydusify/pkg/backend/backend.go b/contrib/nydusify/pkg/backend/backend.go index 292dbd60f25..73c35078996 100644 --- a/contrib/nydusify/pkg/backend/backend.go +++ b/contrib/nydusify/pkg/backend/backend.go @@ -7,6 +7,7 @@ package backend import ( "context" "fmt" + "io" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" @@ -25,6 +26,8 @@ type Backend interface { Finalize(cancel bool) error Check(blobID string) (bool, error) Type() Type + Reader(blobID string) (io.ReadCloser, error) + Size(blobID string) (int64, error) } // TODO: Directly forward blob data to storage backend diff --git a/contrib/nydusify/pkg/backend/oss.go b/contrib/nydusify/pkg/backend/oss.go index 3006eb9530e..8496fef5cc0 100644 --- a/contrib/nydusify/pkg/backend/oss.go +++ b/contrib/nydusify/pkg/backend/oss.go @@ -259,6 +259,26 @@ func (b *OSSBackend) Type() Type { return OssBackend } +func (b *OSSBackend) Reader(blobID string) (io.ReadCloser, error) { + blobID = b.objectPrefix + blobID + rc, err := b.bucket.GetObject(blobID) + return rc, err +} + +func (b *OSSBackend) Size(blobID string) (int64, error) { + blobID = b.objectPrefix + blobID + headers, err := b.bucket.GetObjectMeta(blobID) + if err != nil { + return 0, errors.Wrap(err, "get object size") + } + sizeStr := headers.Get("Content-Length") + size, err := strconv.ParseInt(sizeStr, 10, 0) + if err != nil { + return 0, errors.Wrap(err, "parse content-length header") + } + return size, nil +} + func (b *OSSBackend) remoteID(blobID string) string { return fmt.Sprintf("oss://%s/%s%s", b.bucket.BucketName, b.objectPrefix, blobID) } diff --git a/contrib/nydusify/pkg/backend/registry.go b/contrib/nydusify/pkg/backend/registry.go index 2257cc9b444..674a6e42131 100644 --- a/contrib/nydusify/pkg/backend/registry.go +++ b/contrib/nydusify/pkg/backend/registry.go @@ -2,6 +2,7 @@ package backend import ( "context" + "io" "os" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote" @@ -46,6 +47,14 @@ func (r *Registry) Type() Type { return RegistryBackend } +func (r *Registry) Reader(blobID string) (io.ReadCloser, error) { + panic("not implemented") +} + +func (r *Registry) Size(blobID string) (int64, error) { + panic("not implemented") +} + func newRegistryBackend(rawConfig []byte, remote *remote.Remote) (Backend, error) { return &Registry{remote: remote}, nil } diff --git a/contrib/nydusify/pkg/backend/s3.go b/contrib/nydusify/pkg/backend/s3.go index 27a4fa3eebf..0d9081c2967 100644 --- a/contrib/nydusify/pkg/backend/s3.go +++ b/contrib/nydusify/pkg/backend/s3.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/url" "os" @@ -140,6 +141,14 @@ func (b *S3Backend) Type() Type { return S3backend } +func (b *S3Backend) Reader(blobID string) (io.ReadCloser, error) { + panic("not implemented") +} + +func (b *S3Backend) Size(blobID string) (int64, error) { + panic("not implemented") +} + func (b *S3Backend) existObject(ctx context.Context, objectKey string) (bool, error) { _, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: &b.bucketName, diff --git a/contrib/nydusify/pkg/converter/provider/provider.go b/contrib/nydusify/pkg/converter/provider/provider.go index 1f5e36371be..cdcaad72ea5 100644 --- a/contrib/nydusify/pkg/converter/provider/provider.go +++ b/contrib/nydusify/pkg/converter/provider/provider.go @@ -100,3 +100,7 @@ func (pvd *Provider) Image(ctx context.Context, ref string) (*ocispec.Descriptor func (pvd *Provider) ContentStore() content.Store { return pvd.store } + +func (pvd *Provider) SetContentStore(store content.Store) { + pvd.store = store +} diff --git a/contrib/nydusify/pkg/copier/copier.go b/contrib/nydusify/pkg/copier/copier.go new file mode 100644 index 00000000000..1bc3da249dc --- /dev/null +++ b/contrib/nydusify/pkg/copier/copier.go @@ -0,0 +1,292 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package copier + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/containerd/containerd/content" + containerdErrdefs "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/reference/docker" + "github.com/containerd/nydus-snapshotter/pkg/converter" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/backend" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/tool" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser" + nydusifyUtils "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" + "github.com/goharbor/acceleration-service/pkg/errdefs" + "github.com/goharbor/acceleration-service/pkg/platformutil" + "github.com/goharbor/acceleration-service/pkg/remote" + "github.com/goharbor/acceleration-service/pkg/utils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type Opt struct { + WorkDir string + NydusImagePath string + + Source string + Target string + + SourceInsecure bool + TargetInsecure bool + + SourceBackendType string + SourceBackendConfig string + + TargetBackendType string + TargetBackendConfig string + + AllPlatforms bool + Platforms string +} + +type output struct { + Blobs []string +} + +func hosts(opt Opt) remote.HostFunc { + maps := map[string]bool{ + opt.Source: opt.SourceInsecure, + opt.Target: opt.TargetInsecure, + } + return func(ref string) (remote.CredentialFunc, bool, error) { + return remote.NewDockerConfigCredFunc(), maps[ref], nil + } +} + +func getPushWriter(ctx context.Context, pvd *provider.Provider, desc ocispec.Descriptor, opt Opt) (content.Writer, error) { + resolver, err := pvd.Resolver(opt.Target) + if err != nil { + return nil, errors.Wrap(err, "get resolver") + } + pusher, err := resolver.Pusher(ctx, opt.Target) + if err != nil { + return nil, errors.Wrap(err, "create pusher") + } + writer, err := pusher.Push(ctx, desc) + if err != nil { + if containerdErrdefs.IsAlreadyExists(err) { + return nil, nil + } + return nil, err + } + return writer, nil +} + +func pushBlobFromBackend( + ctx context.Context, pvd *provider.Provider, backend backend.Backend, src ocispec.Descriptor, opt Opt, +) ([]ocispec.Descriptor, *ocispec.Descriptor, error) { + if src.MediaType != ocispec.MediaTypeImageManifest && src.MediaType != images.MediaTypeDockerSchema2Manifest { + return nil, nil, fmt.Errorf("unsupported media type %s", src.MediaType) + } + manifest := ocispec.Manifest{} + if _, err := utils.ReadJSON(ctx, pvd.ContentStore(), &manifest, src); err != nil { + return nil, nil, errors.Wrap(err, "read manifest from store") + } + bootstrapDesc := parser.FindNydusBootstrapDesc(&manifest) + if bootstrapDesc == nil { + return nil, nil, nil + } + ra, err := pvd.ContentStore().ReaderAt(ctx, *bootstrapDesc) + if err != nil { + return nil, nil, errors.Wrap(err, "prepare reading bootstrap") + } + bootstrapPath := filepath.Join(opt.WorkDir, "bootstrap.tgz") + if err := nydusifyUtils.UnpackFile(io.NewSectionReader(ra, 0, ra.Size()), nydusifyUtils.BootstrapFileNameInLayer, bootstrapPath); err != nil { + return nil, nil, errors.Wrap(err, "unpack bootstrap layer") + } + outputPath := filepath.Join(opt.WorkDir, "output.json") + builder := tool.NewBuilder(opt.NydusImagePath) + if err := builder.Check(tool.BuilderOption{ + BootstrapPath: bootstrapPath, + DebugOutputPath: outputPath, + }); err != nil { + return nil, nil, errors.Wrap(err, "check bootstrap") + } + var out output + bytes, err := os.ReadFile(outputPath) + if err != nil { + return nil, nil, errors.Wrap(err, "read output file") + } + if err := json.Unmarshal(bytes, &out); err != nil { + return nil, nil, errors.Wrap(err, "unmarshal output json") + } + + // Deduplicate the blobs for avoiding uploading repeatedly. + blobIDs := []string{} + blobIDMap := map[string]bool{} + for _, blobID := range out.Blobs { + if blobIDMap[blobID] { + continue + } + blobIDs = append(blobIDs, blobID) + blobIDMap[blobID] = true + } + + eg, ctx := errgroup.WithContext(ctx) + descs := make([]ocispec.Descriptor, len(blobIDs)) + for idx := range blobIDs { + func(idx int) { + eg.Go(func() error { + blobID := blobIDs[idx] + size, err := backend.Size(blobID) + if err != nil { + return errors.Wrap(err, "get blob size") + } + rc, err := backend.Reader(blobID) + if err != nil { + return errors.Wrap(err, "get blob reader") + } + defer rc.Close() + descs[idx] = ocispec.Descriptor{ + Digest: digest.Digest("sha256:" + blobID), + Size: size, + MediaType: converter.MediaTypeNydusBlob, + Annotations: map[string]string{ + converter.LayerAnnotationNydusBlob: "true", + }, + } + writer, err := getPushWriter(ctx, pvd, descs[idx], opt) + if err != nil { + if errdefs.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + writer, err = getPushWriter(ctx, pvd, descs[idx], opt) + } + if err != nil { + return errors.Wrap(err, "get push writer") + } + } + if writer != nil { + defer writer.Close() + if _, err := io.Copy(writer, rc); err != nil { + return errors.Wrap(err, "push blob") + } + } + return nil + }) + }(idx) + } + + if err := eg.Wait(); err != nil { + return nil, nil, errors.Wrap(err, "push blobs") + } + + manifest.Layers = append(descs, manifest.Layers...) + target, err := utils.WriteJSON(ctx, pvd.ContentStore(), &manifest, src, opt.Target, nil) + if err != nil { + return nil, nil, errors.Wrap(err, "write json") + } + + return descs, target, nil +} + +func Copy(ctx context.Context, opt Opt) error { + platformMC, err := platformutil.ParsePlatforms(opt.AllPlatforms, opt.Platforms) + if err != nil { + return err + } + + var bkd backend.Backend + if opt.SourceBackendType != "" { + bkd, err = backend.NewBackend(opt.SourceBackendType, []byte(opt.SourceBackendConfig), nil) + if err != nil { + return errors.Wrapf(err, "new backend") + } + } + + if _, err := os.Stat(opt.WorkDir); err != nil { + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(opt.WorkDir, 0755); err != nil { + return errors.Wrap(err, "prepare work directory") + } + // We should only clean up when the work directory not exists + // before, otherwise it may delete user data by mistake. + defer os.RemoveAll(opt.WorkDir) + } else { + return errors.Wrap(err, "stat work directory") + } + } + tmpDir, err := os.MkdirTemp(opt.WorkDir, "nydusify-") + if err != nil { + return errors.Wrap(err, "create temp directory") + } + pvd, err := provider.New(tmpDir, hosts(opt), platformMC) + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + sourceNamed, err := docker.ParseDockerRef(opt.Source) + if err != nil { + return errors.Wrap(err, "parse source reference") + } + targetNamed, err := docker.ParseDockerRef(opt.Target) + if err != nil { + return errors.Wrap(err, "parse target reference") + } + source := sourceNamed.String() + target := targetNamed.String() + + logrus.Infof("pulling source image %s", source) + if err := pvd.Pull(ctx, source); err != nil { + if errdefs.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + if err := pvd.Pull(ctx, source); err != nil { + return errors.Wrap(err, "try to pull image") + } + } else { + return errors.Wrap(err, "pull source image") + } + } + logrus.Infof("pulled source image %s", source) + + sourceImage, err := pvd.Image(ctx, source) + if err != nil { + return errors.Wrap(err, "find image from store") + } + targetImage := sourceImage + if bkd != nil { + logrus.Infof("pushing blob layers from backend") + descs, _targetImage, err := pushBlobFromBackend(ctx, pvd, bkd, *sourceImage, opt) + if err != nil { + return errors.Wrap(err, "get resolver") + } + if _targetImage == nil { + logrus.Warnf("%s is not a nydus image", source) + } else { + targetImage = _targetImage + store := newStore(pvd.ContentStore(), descs) + pvd.SetContentStore(store) + logrus.Infof("pushed blob layers from backend") + } + } + + logrus.Infof("pushing target image %s", target) + if err := pvd.Push(ctx, *targetImage, target); err != nil { + if errdefs.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + if err := pvd.Push(ctx, *sourceImage, target); err != nil { + return errors.Wrap(err, "try to push image") + } + } else { + return errors.Wrap(err, "push target image") + } + } + logrus.Infof("pushed target image %s", target) + + return nil +} diff --git a/contrib/nydusify/pkg/copier/store.go b/contrib/nydusify/pkg/copier/store.go new file mode 100644 index 00000000000..790f3b7a28b --- /dev/null +++ b/contrib/nydusify/pkg/copier/store.go @@ -0,0 +1,45 @@ +// Copyright 2023 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package copier + +import ( + "context" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type store struct { + content.Store + remotes []ocispec.Descriptor +} + +func newStore(base content.Store, remotes []ocispec.Descriptor) *store { + return &store{ + Store: base, + remotes: remotes, + } +} + +func (s *store) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { + info, err := s.Store.Info(ctx, dgst) + if err != nil { + if !errdefs.IsNotFound(err) { + return content.Info{}, err + } + for _, desc := range s.remotes { + if desc.Digest == dgst { + return content.Info{ + Digest: desc.Digest, + Size: desc.Size, + }, nil + } + } + return content.Info{}, err + } + return info, nil +} diff --git a/contrib/nydusify/pkg/packer/pusher_test.go b/contrib/nydusify/pkg/packer/pusher_test.go index 2128414b4fd..178dd179d18 100644 --- a/contrib/nydusify/pkg/packer/pusher_test.go +++ b/contrib/nydusify/pkg/packer/pusher_test.go @@ -2,6 +2,7 @@ package packer import ( "context" + "io" "os" "path/filepath" "testing" @@ -35,6 +36,14 @@ func (m *mockBackend) Type() backend.Type { return backend.OssBackend } +func (m *mockBackend) Reader(blobID string) (io.ReadCloser, error) { + panic("not implemented") +} + +func (m *mockBackend) Size(blobID string) (int64, error) { + panic("not implemented") +} + func Test_parseBackendConfig(t *testing.T) { cfg, err := ParseBackendConfig("oss", filepath.Join("testdata", "backend-config.json")) assert.Nil(t, err)