nydusify: introduce copy subcommand

contrib/nydusify/cmd/nydusify.go

@@ -26,6 +26,7 @@ import (
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/chunkdict/generator"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/copier"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
@@ -68,27 +69,27 @@ func parseBackendConfig(backendConfigJSON, backendConfigFile string) (string, er
 	return backendConfigJSON, nil
 }
 
-func getBackendConfig(c *cli.Context, required bool) (string, string, error) {
-	backendType := c.String("backend-type")
+func getBackendConfig(c *cli.Context, suffix string, required bool) (string, string, error) {
+	backendType := c.String(suffix + "backend-type")
 	if backendType == "" {
 		if required {
-			return "", "", errors.Errorf("backend type is empty, please specify option '--backend-type'")
+			return "", "", errors.Errorf("backend type is empty, please specify option '--%sbackend-type'", suffix)
 		}
 		return "", "", nil
 	}
 
 	possibleBackendTypes := []string{"oss", "s3"}
 	if !isPossibleValue(possibleBackendTypes, backendType) {
-		return "", "", fmt.Errorf("--backend-type should be one of %v", possibleBackendTypes)
+		return "", "", fmt.Errorf("--%sbackend-type should be one of %v", suffix, possibleBackendTypes)
 	}
 
 	backendConfig, err := parseBackendConfig(
-		c.String("backend-config"), c.String("backend-config-file"),
+		c.String(suffix+"backend-config"), c.String(suffix+"backend-config-file"),
 	)
 	if err != nil {
 		return "", "", err
 	} else if (backendType == "oss" || backendType == "s3") && strings.TrimSpace(backendConfig) == "" {
-		return "", "", errors.Errorf("backend configuration is empty, please specify option '--backend-config'")
+		return "", "", errors.Errorf("backend configuration is empty, please specify option '--%sbackend-config'", suffix)
 	}
 
 	return backendType, backendConfig, nil
@@ -434,7 +435,7 @@ func main() {
 					return err
 				}
 
-				backendType, backendConfig, err := getBackendConfig(c, false)
+				backendType, backendConfig, err := getBackendConfig(c, "", false)
 				if err != nil {
 					return err
 				}
@@ -610,7 +611,7 @@ func main() {
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
 
-				backendType, backendConfig, err := getBackendConfig(c, false)
+				backendType, backendConfig, err := getBackendConfig(c, "", false)
 				if err != nil {
 					return err
 				}
@@ -769,7 +770,7 @@ func main() {
 			Action: func(c *cli.Context) error {
 				setupLogLevel(c)
 
-				backendType, backendConfig, err := getBackendConfig(c, false)
+				backendType, backendConfig, err := getBackendConfig(c, "", false)
 				if err != nil {
 					return err
 				} else if backendConfig == "" {
@@ -945,7 +946,7 @@ func main() {
 
 				// if backend-push is specified, we should make sure backend-config-file exists
 				if c.Bool("backend-push") || c.Bool("compact") {
-					_backendType, _backendConfig, err := getBackendConfig(c, true)
+					_backendType, _backendConfig, err := getBackendConfig(c, "", true)
 					if err != nil {
 						return err
 					}
@@ -985,6 +986,106 @@ func main() {
 				return nil
 			},
 		},
+		{
+			Name:  "copy",
+			Usage: "Copy an image from source to target",
+			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name:     "source",
+					Required: true,
+					Usage:    "Source image reference",
+					EnvVars:  []string{"SOURCE"},
+				},
+				&cli.StringFlag{
+					Name:     "target",
+					Required: false,
+					Usage:    "Target image reference",
+					EnvVars:  []string{"TARGET"},
+				},
+				&cli.BoolFlag{
+					Name:     "source-insecure",
+					Required: false,
+					Usage:    "Skip verifying server certs for HTTPS source registry",
+					EnvVars:  []string{"SOURCE_INSECURE"},
+				},
+				&cli.BoolFlag{
+					Name:     "target-insecure",
+					Required: false,
+					Usage:    "Skip verifying server certs for HTTPS target registry",
+					EnvVars:  []string{"TARGET_INSECURE"},
+				},
+
+				&cli.StringFlag{
+					Name:    "source-backend-type",
+					Value:   "",
+					Usage:   "Type of storage backend, possible values: 'oss', 's3'",
+					EnvVars: []string{"BACKEND_TYPE"},
+				},
+				&cli.StringFlag{
+					Name:    "source-backend-config",
+					Value:   "",
+					Usage:   "Json configuration string for storage backend",
+					EnvVars: []string{"BACKEND_CONFIG"},
+				},
+				&cli.PathFlag{
+					Name:      "source-backend-config-file",
+					Value:     "",
+					TakesFile: true,
+					Usage:     "Json configuration file for storage backend",
+					EnvVars:   []string{"BACKEND_CONFIG_FILE"},
+				},
+
+				&cli.BoolFlag{
+					Name:  "all-platforms",
+					Value: false,
+					Usage: "Copy images for all platforms, conflicts with --platform",
+				},
+				&cli.StringFlag{
+					Name:  "platform",
+					Value: "linux/" + runtime.GOARCH,
+					Usage: "Copy images for specific platforms, for example: 'linux/amd64,linux/arm64'",
+				},
+
+				&cli.StringFlag{
+					Name:    "work-dir",
+					Value:   "./tmp",
+					Usage:   "Working directory for image copy",
+					EnvVars: []string{"WORK_DIR"},
+				},
+				&cli.StringFlag{
+					Name:    "nydus-image",
+					Value:   "nydus-image",
+					Usage:   "Path to the nydus-image binary, default to search in PATH",
+					EnvVars: []string{"NYDUS_IMAGE"},
+				},
+			},
+			Action: func(c *cli.Context) error {
+				setupLogLevel(c)
+
+				sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false)
+				if err != nil {
+					return err
+				}
+
+				opt := copier.Opt{
+					WorkDir:        c.String("work-dir"),
+					NydusImagePath: c.String("nydus-image"),
+
+					Source:         c.String("source"),
+					Target:         c.String("target"),
+					SourceInsecure: c.Bool("source-insecure"),
+					TargetInsecure: c.Bool("target-insecure"),
+
+					SourceBackendType:   sourceBackendType,
+					SourceBackendConfig: sourceBackendConfig,
+
+					AllPlatforms: c.Bool("all-platforms"),
+					Platforms:    c.String("platform"),
+				}
+
+				return copier.Copy(context.Background(), opt)
+			},
+		},
 	}
 
 	if !utils.IsSupportedArch(runtime.GOARCH) {

contrib/nydusify/go.mod

@@ -10,8 +10,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
 	github.com/containerd/containerd v1.7.2
+	github.com/containerd/nydus-snapshotter v0.10.0
 	github.com/docker/cli v23.0.3+incompatible
 	github.com/docker/distribution v2.8.2+incompatible
+	github.com/dustin/go-humanize v1.0.1
 	github.com/goharbor/acceleration-service v0.2.6
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-hclog v1.3.1
@@ -53,7 +55,6 @@ require (
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/containerd/continuity v0.4.1 // indirect
 	github.com/containerd/fifo v1.1.0 // indirect
-	github.com/containerd/nydus-snapshotter v0.10.0 // indirect
 	github.com/containerd/stargz-snapshotter v0.14.3 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containerd/ttrpc v1.2.2 // indirect
@@ -65,7 +66,6 @@ require (
 	github.com/docker/docker v23.0.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

contrib/nydusify/pkg/backend/backend.go

@@ -7,6 +7,7 @@ package backend
 import (
 	"context"
 	"fmt"
+	"io"
 
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
@@ -25,6 +26,8 @@ type Backend interface {
 	Finalize(cancel bool) error
 	Check(blobID string) (bool, error)
 	Type() Type
+	Reader(blobID string) (io.ReadCloser, error)
+	Size(blobID string) (int64, error)
 }
 
 // TODO: Directly forward blob data to storage backend

contrib/nydusify/pkg/backend/oss.go

@@ -259,6 +259,26 @@ func (b *OSSBackend) Type() Type {
 	return OssBackend
 }
 
+func (b *OSSBackend) Reader(blobID string) (io.ReadCloser, error) {
+	blobID = b.objectPrefix + blobID
+	rc, err := b.bucket.GetObject(blobID)
+	return rc, err
+}
+
+func (b *OSSBackend) Size(blobID string) (int64, error) {
+	blobID = b.objectPrefix + blobID
+	headers, err := b.bucket.GetObjectMeta(blobID)
+	if err != nil {
+		return 0, errors.Wrap(err, "get object size")
+	}
+	sizeStr := headers.Get("Content-Length")
+	size, err := strconv.ParseInt(sizeStr, 10, 0)
+	if err != nil {
+		return 0, errors.Wrap(err, "parse content-length header")
+	}
+	return size, nil
+}
+
 func (b *OSSBackend) remoteID(blobID string) string {
 	return fmt.Sprintf("oss://%s/%s%s", b.bucket.BucketName, b.objectPrefix, blobID)
 }

contrib/nydusify/pkg/backend/registry.go

@@ -2,6 +2,7 @@ package backend
 
 import (
 	"context"
+	"io"
 	"os"
 
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
@@ -46,6 +47,14 @@ func (r *Registry) Type() Type {
 	return RegistryBackend
 }
 
+func (r *Registry) Reader(blobID string) (io.ReadCloser, error) {
+	panic("not implemented")
+}
+
+func (r *Registry) Size(blobID string) (int64, error) {
+	panic("not implemented")
+}
+
 func newRegistryBackend(rawConfig []byte, remote *remote.Remote) (Backend, error) {
 	return &Registry{remote: remote}, nil
 }

contrib/nydusify/pkg/backend/s3.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -159,6 +160,27 @@ func (b *S3Backend) blobObjectKey(blobID string) string {
 	return b.objectPrefix + blobID
 }
 
+func (b *S3Backend) Reader(blobID string) (io.ReadCloser, error) {
+	objectKey := b.blobObjectKey(blobID)
+	output, err := b.client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: &b.bucketName,
+		Key:    &objectKey,
+	})
+	return output.Body, err
+}
+
+func (b *S3Backend) Size(blobID string) (int64, error) {
+	objectKey := b.blobObjectKey(blobID)
+	output, err := b.client.GetObjectAttributes(context.TODO(), &s3.GetObjectAttributesInput{
+		Bucket: &b.bucketName,
+		Key:    &objectKey,
+	})
+	if err != nil {
+		return 0, errors.Wrap(err, "get object size")
+	}
+	return output.ObjectSize, nil
+}
+
 func (b *S3Backend) remoteID(blobObjectKey string) string {
 	remoteURL, _ := url.Parse(b.endpointWithScheme)
 	remoteURL.Path = path.Join(remoteURL.Path, b.bucketName, blobObjectKey)

contrib/nydusify/pkg/converter/provider/provider.go

@@ -18,6 +18,8 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+var LayerConcurrentLimit = 5
+
 type Provider struct {
 	mutex        sync.Mutex
 	usePlainHTTP bool
@@ -59,8 +61,9 @@ func (pvd *Provider) Pull(ctx context.Context, ref string) error {
 		return err
 	}
 	rc := &containerd.RemoteContext{
-		Resolver:        resolver,
-		PlatformMatcher: pvd.platformMC,
+		Resolver:               resolver,
+		PlatformMatcher:        pvd.platformMC,
+		MaxConcurrentDownloads: LayerConcurrentLimit,
 	}
 
 	img, err := fetch(ctx, pvd.store, rc, ref, 0)
@@ -81,8 +84,9 @@ func (pvd *Provider) Push(ctx context.Context, desc ocispec.Descriptor, ref stri
 		return err
 	}
 	rc := &containerd.RemoteContext{
-		Resolver:        resolver,
-		PlatformMatcher: pvd.platformMC,
+		Resolver:                    resolver,
+		PlatformMatcher:             pvd.platformMC,
+		MaxConcurrentUploadedLayers: LayerConcurrentLimit,
 	}
 
 	return push(ctx, pvd.store, rc, desc, ref)
@@ -100,3 +104,7 @@ func (pvd *Provider) Image(ctx context.Context, ref string) (*ocispec.Descriptor
 func (pvd *Provider) ContentStore() content.Store {
 	return pvd.store
 }
+
+func (pvd *Provider) SetContentStore(store content.Store) {
+	pvd.store = store
+}