Skip to content

Commit

Permalink
nydusify: introduce copy subcommand
Browse files Browse the repository at this point in the history 
`nydusify copy` copies an image from source registry to target
registry, it also supports to specify a source backend storage.

Signed-off-by: Yan Song <[email protected]>
  • Loading branch information
@imeoer
imeoer committed Aug 23, 2023
1 parent 156ba6a commit f587b68
Show file tree
Hide file tree
Showing 10 changed files with 622 additions and 16 deletions.
121 changes: 111 additions & 10 deletions contrib/nydusify/cmd/nydusify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/chunkdict/generator"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/copier"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
Expand Down Expand Up @@ -68,27 +69,27 @@ func parseBackendConfig(backendConfigJSON, backendConfigFile string) (string, er
return backendConfigJSON, nil
}

func getBackendConfig(c *cli.Context, required bool) (string, string, error) {
backendType := c.String("backend-type")
func getBackendConfig(c *cli.Context, suffix string, required bool) (string, string, error) {
backendType := c.String(suffix + "backend-type")
if backendType == "" {
if required {
return "", "", errors.Errorf("backend type is empty, please specify option '--backend-type'")
return "", "", errors.Errorf("backend type is empty, please specify option '--%sbackend-type'", suffix)
}
return "", "", nil
}

possibleBackendTypes := []string{"oss", "s3"}
if !isPossibleValue(possibleBackendTypes, backendType) {
return "", "", fmt.Errorf("--backend-type should be one of %v", possibleBackendTypes)
return "", "", fmt.Errorf("--%sbackend-type should be one of %v", suffix, possibleBackendTypes)
}

backendConfig, err := parseBackendConfig(
c.String("backend-config"), c.String("backend-config-file"),
c.String(suffix+"backend-config"), c.String(suffix+"backend-config-file"),
)
if err != nil {
return "", "", err
} else if (backendType == "oss" || backendType == "s3") && strings.TrimSpace(backendConfig) == "" {
return "", "", errors.Errorf("backend configuration is empty, please specify option '--backend-config'")
return "", "", errors.Errorf("backend configuration is empty, please specify option '--%sbackend-config'", suffix)
}

return backendType, backendConfig, nil
Expand Down Expand Up @@ -434,7 +435,7 @@ func main() {
return err
}

backendType, backendConfig, err := getBackendConfig(c, false)
backendType, backendConfig, err := getBackendConfig(c, "", false)
if err != nil {
return err
}
Expand Down Expand Up @@ -610,7 +611,7 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, false)
backendType, backendConfig, err := getBackendConfig(c, "", false)
if err != nil {
return err
}
Expand Down Expand Up @@ -769,7 +770,7 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, false)
backendType, backendConfig, err := getBackendConfig(c, "", false)
if err != nil {
return err
} else if backendConfig == "" {
Expand Down Expand Up @@ -945,7 +946,7 @@ func main() {

// if backend-push is specified, we should make sure backend-config-file exists
if c.Bool("backend-push") || c.Bool("compact") {
_backendType, _backendConfig, err := getBackendConfig(c, true)
_backendType, _backendConfig, err := getBackendConfig(c, "", true)
if err != nil {
return err
}
Expand Down Expand Up @@ -985,6 +986,106 @@ func main() {
return nil
},
},
{
Name: "copy",
Usage: "Copy an image from source to target",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "source",
Required: true,
Usage: "Source image reference",
EnvVars: []string{"SOURCE"},
},
&cli.StringFlag{
Name: "target",
Required: false,
Usage: "Target image reference",
EnvVars: []string{"TARGET"},
},
&cli.BoolFlag{
Name: "source-insecure",
Required: false,
Usage: "Skip verifying server certs for HTTPS source registry",
EnvVars: []string{"SOURCE_INSECURE"},
},
&cli.BoolFlag{
Name: "target-insecure",
Required: false,
Usage: "Skip verifying server certs for HTTPS target registry",
EnvVars: []string{"TARGET_INSECURE"},
},

&cli.StringFlag{
Name: "source-backend-type",
Value: "",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "source-backend-config",
Value: "",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "source-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
EnvVars: []string{"BACKEND_CONFIG_FILE"},
},

&cli.BoolFlag{
Name: "all-platforms",
Value: false,
Usage: "Convert images for all platforms, conflicts with --platform",
},
&cli.StringFlag{
Name: "platform",
Value: "linux/" + runtime.GOARCH,
Usage: "Convert images for specific platforms, for example: 'linux/amd64,linux/arm64'",
},

&cli.StringFlag{
Name: "work-dir",
Value: "./tmp",
Usage: "Working directory for image conversion",
EnvVars: []string{"WORK_DIR"},
},
&cli.StringFlag{
Name: "nydus-image",
Value: "nydus-image",
Usage: "Path to the nydus-image binary, default to search in PATH",
EnvVars: []string{"NYDUS_IMAGE"},
},
},
Action: func(c *cli.Context) error {
setupLogLevel(c)

sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false)
if err != nil {
return err
}

opt := copier.Opt{
WorkDir: c.String("work-dir"),
NydusImagePath: c.String("nydus-image"),

Source: c.String("source"),
Target: c.String("target"),
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),

SourceBackendType: sourceBackendType,
SourceBackendConfig: sourceBackendConfig,

AllPlatforms: c.Bool("all-platforms"),
Platforms: c.String("platform"),
}

return copier.Copy(context.Background(), opt)
},
},
}

if !utils.IsSupportedArch(runtime.GOARCH) {
Expand Down
4 changes: 2 additions & 2 deletions contrib/nydusify/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@ require (
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
github.com/containerd/containerd v1.7.2
github.com/containerd/nydus-snapshotter v0.10.0
github.com/docker/cli v23.0.3+incompatible
github.com/docker/distribution v2.8.2+incompatible
github.com/dustin/go-humanize v1.0.1
github.com/goharbor/acceleration-service v0.2.6
github.com/google/uuid v1.3.0
github.com/hashicorp/go-hclog v1.3.1
Expand Down Expand Up @@ -53,7 +55,6 @@ require (
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/continuity v0.4.1 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/containerd/nydus-snapshotter v0.10.0 // indirect
github.com/containerd/stargz-snapshotter v0.14.3 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/containerd/ttrpc v1.2.2 // indirect
Expand All @@ -65,7 +66,6 @@ require (
github.com/docker/docker v23.0.3+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand Down
3 changes: 3 additions & 0 deletions contrib/nydusify/pkg/backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package backend
import (
"context"
"fmt"
"io"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
Expand All @@ -25,6 +26,8 @@ type Backend interface {
Finalize(cancel bool) error
Check(blobID string) (bool, error)
Type() Type
Reader(blobID string) (io.ReadCloser, error)
Size(blobID string) (int64, error)
}

// TODO: Directly forward blob data to storage backend
Expand Down
20 changes: 20 additions & 0 deletions contrib/nydusify/pkg/backend/oss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,26 @@ func (b *OSSBackend) Type() Type {
return OssBackend
}

func (b *OSSBackend) Reader(blobID string) (io.ReadCloser, error) {
blobID = b.objectPrefix + blobID
rc, err := b.bucket.GetObject(blobID)
return rc, err
}

func (b *OSSBackend) Size(blobID string) (int64, error) {
blobID = b.objectPrefix + blobID
headers, err := b.bucket.GetObjectMeta(blobID)
if err != nil {
return 0, errors.Wrap(err, "get object size")
}
sizeStr := headers.Get("Content-Length")
size, err := strconv.ParseInt(sizeStr, 10, 0)
if err != nil {
return 0, errors.Wrap(err, "parse content-length header")
}
return size, nil
}

func (b *OSSBackend) remoteID(blobID string) string {
return fmt.Sprintf("oss://%s/%s%s", b.bucket.BucketName, b.objectPrefix, blobID)
}
9 changes: 9 additions & 0 deletions contrib/nydusify/pkg/backend/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package backend

import (
"context"
"io"
"os"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
Expand Down Expand Up @@ -46,6 +47,14 @@ func (r *Registry) Type() Type {
return RegistryBackend
}

func (r *Registry) Reader(blobID string) (io.ReadCloser, error) {
panic("not implemented")
}

func (r *Registry) Size(blobID string) (int64, error) {
panic("not implemented")
}

func newRegistryBackend(rawConfig []byte, remote *remote.Remote) (Backend, error) {
return &Registry{remote: remote}, nil
}
22 changes: 22 additions & 0 deletions contrib/nydusify/pkg/backend/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
Expand Down Expand Up @@ -159,6 +160,27 @@ func (b *S3Backend) blobObjectKey(blobID string) string {
return b.objectPrefix + blobID
}

func (b *S3Backend) Reader(blobID string) (io.ReadCloser, error) {
objectKey := b.blobObjectKey(blobID)
output, err := b.client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &b.bucketName,
Key: &objectKey,
})
return output.Body, err
}

func (b *S3Backend) Size(blobID string) (int64, error) {
objectKey := b.blobObjectKey(blobID)
output, err := b.client.GetObjectAttributes(context.TODO(), &s3.GetObjectAttributesInput{
Bucket: &b.bucketName,
Key: &objectKey,
})
if err != nil {
return 0, errors.Wrap(err, "get object size")
}
return output.ObjectSize, nil
}

func (b *S3Backend) remoteID(blobObjectKey string) string {
remoteURL, _ := url.Parse(b.endpointWithScheme)
remoteURL.Path = path.Join(remoteURL.Path, b.bucketName, blobObjectKey)
Expand Down
16 changes: 12 additions & 4 deletions contrib/nydusify/pkg/converter/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

var LayerConcurrentLimit = 5

type Provider struct {
mutex sync.Mutex
usePlainHTTP bool
Expand Down Expand Up @@ -59,8 +61,9 @@ func (pvd *Provider) Pull(ctx context.Context, ref string) error {
return err
}
rc := &containerd.RemoteContext{
Resolver: resolver,
PlatformMatcher: pvd.platformMC,
Resolver: resolver,
PlatformMatcher: pvd.platformMC,
MaxConcurrentDownloads: LayerConcurrentLimit,
}

img, err := fetch(ctx, pvd.store, rc, ref, 0)
Expand All @@ -81,8 +84,9 @@ func (pvd *Provider) Push(ctx context.Context, desc ocispec.Descriptor, ref stri
return err
}
rc := &containerd.RemoteContext{
Resolver: resolver,
PlatformMatcher: pvd.platformMC,
Resolver: resolver,
PlatformMatcher: pvd.platformMC,
MaxConcurrentUploadedLayers: LayerConcurrentLimit,
}

return push(ctx, pvd.store, rc, desc, ref)
Expand All @@ -100,3 +104,7 @@ func (pvd *Provider) Image(ctx context.Context, ref string) (*ocispec.Descriptor
func (pvd *Provider) ContentStore() content.Store {
return pvd.store
}

func (pvd *Provider) SetContentStore(store content.Store) {
pvd.store = store
}
Loading

0 comments on commit f587b68

Please sign in to comment.