- support download blobs in rpull

- add tar header for blobs in userspace convertor

Signed-off-by: yuchen.cc <[email protected]>

cmd/convertor/builder/builder_engine.go

@@ -27,6 +27,7 @@ import (
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 type BuilderEngineType int
@@ -125,6 +126,7 @@ func (e *builderEngineBase) uploadManifestAndConfig(ctx context.Context) error {
 	if err = uploadBytes(ctx, e.pusher, e.manifest.Config, cbuf); err != nil {
 		return errors.Wrapf(err, "failed to upload config")
 	}
+	logrus.Infof("config uploaded")
 
 	e.manifest.MediaType = e.mediaTypeManifest()
 	cbuf, err = json.Marshal(e.manifest)
@@ -140,6 +142,8 @@ func (e *builderEngineBase) uploadManifestAndConfig(ctx context.Context) error {
 	if err = uploadBytes(ctx, e.pusher, manifestDesc, cbuf); err != nil {
 		return errors.Wrapf(err, "failed to upload manifest")
 	}
+	logrus.Infof("manifest uploaded")
+
 	return nil
 }
 

cmd/convertor/builder/overlaybd_builder.go

@@ -17,8 +17,11 @@
 package builder
 
 import (
+	"archive/tar"
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path"
@@ -29,6 +32,7 @@ import (
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -97,7 +101,7 @@ func (e *overlaybdBuilderEngine) UploadLayer(ctx context.Context, idx int) error
 	if err != nil {
 		return errors.Wrapf(err, "failed to get descriptor for layer %d", idx)
 	}
-	desc.MediaType = e.mediaTypeImageLayerGzip()
+	desc.MediaType = e.mediaTypeImageLayer()
 	desc.Annotations = map[string]string{
 		label.OverlayBDBlobDigest: desc.Digest.String(),
 		label.OverlayBDBlobSize:   fmt.Sprintf("%d", desc.Size),
@@ -114,17 +118,9 @@ func (e *overlaybdBuilderEngine) UploadImage(ctx context.Context) error {
 		e.manifest.Layers[idx] = e.overlaybdLayers[idx]
 		e.config.RootFS.DiffIDs[idx] = e.overlaybdLayers[idx].Digest
 	}
-	baseDesc := specs.Descriptor{
-		MediaType: e.mediaTypeImageLayer(),
-		Digest:    "sha256:c3a417552a6cf9ffa959b541850bab7d7f08f4255425bf8b48c85f7b36b378d9",
-		Size:      4737695,
-		Annotations: map[string]string{
-			label.OverlayBDBlobDigest: "sha256:c3a417552a6cf9ffa959b541850bab7d7f08f4255425bf8b48c85f7b36b378d9",
-			label.OverlayBDBlobSize:   "4737695",
-		},
-	}
-	if err := uploadBlob(ctx, e.pusher, overlaybdBaseLayer, baseDesc); err != nil {
-		return errors.Wrapf(err, "failed to upload baselayer %q", overlaybdBaseLayer)
+	baseDesc, err := e.uploadBaseLayer(ctx)
+	if err != nil {
+		return err
 	}
 	e.manifest.Layers = append([]specs.Descriptor{baseDesc}, e.manifest.Layers...)
 	e.config.RootFS.DiffIDs = append([]digest.Digest{baseDesc.Digest}, e.config.RootFS.DiffIDs...)
@@ -135,6 +131,58 @@ func (e *overlaybdBuilderEngine) Cleanup() {
 	os.RemoveAll(e.workDir)
 }
 
+func (e *overlaybdBuilderEngine) uploadBaseLayer(ctx context.Context) (specs.Descriptor, error) {
+	// add baselayer with tar header
+	tarFile := path.Join(e.workDir, "ext4_64.tar")
+	fdes, err := os.Create(tarFile)
+	if err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to create file %s", tarFile)
+	}
+	digester := digest.Canonical.Digester()
+	countWriter := &writeCountWrapper{w: io.MultiWriter(fdes, digester.Hash())}
+	tarWriter := tar.NewWriter(countWriter)
+
+	fsrc, err := os.Open(overlaybdBaseLayer)
+	if err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to open %s", overlaybdBaseLayer)
+	}
+	fstat, err := os.Stat(overlaybdBaseLayer)
+	if err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to get info of %s", overlaybdBaseLayer)
+	}
+
+	if err := tarWriter.WriteHeader(&tar.Header{
+		Name:     commitFile,
+		Mode:     0444,
+		Size:     fstat.Size(),
+		Typeflag: tar.TypeReg,
+	}); err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to write tar header")
+	}
+	if _, err := io.Copy(tarWriter, bufio.NewReader(fsrc)); err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to copy IO")
+	}
+	if err = tarWriter.Close(); err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to close tar file")
+	}
+
+	baseDesc := specs.Descriptor{
+		MediaType: e.mediaTypeImageLayer(),
+		Digest:    digester.Digest(),
+		Size:      countWriter.c,
+		Annotations: map[string]string{
+			"containerd.io/snapshot/overlaybd/blob-digest": digester.Digest().String(),
+			"containerd.io/snapshot/overlaybd/blob-size":   fmt.Sprintf("%d", countWriter.c),
+		},
+	}
+	if err = uploadBlob(ctx, e.pusher, tarFile, baseDesc); err != nil {
+		return specs.Descriptor{}, errors.Wrapf(err, "failed to upload baselayer")
+	}
+	logrus.Infof("baselayer uploaded")
+
+	return baseDesc, nil
+}
+
 func (e *overlaybdBuilderEngine) getLayerDir(idx int) string {
 	return path.Join(e.workDir, fmt.Sprintf("%04d_", idx)+e.manifest.Layers[idx].Digest.String())
 }
@@ -169,7 +217,7 @@ func (e *overlaybdBuilderEngine) apply(ctx context.Context, dir string) error {
 func (e *overlaybdBuilderEngine) commit(ctx context.Context, dir string) error {
 	binpath := filepath.Join("/opt/overlaybd/bin", "overlaybd-commit")
 
-	out, err := exec.CommandContext(ctx, binpath, "-z",
+	out, err := exec.CommandContext(ctx, binpath, "-z", "-t",
 		path.Join(dir, "writable_data"),
 		path.Join(dir, "writable_index"),
 		path.Join(dir, commitFile),
@@ -179,3 +227,14 @@ func (e *overlaybdBuilderEngine) commit(ctx context.Context, dir string) error {
 	}
 	return nil
 }
+
+type writeCountWrapper struct {
+	w io.Writer
+	c int64
+}
+
+func (wc *writeCountWrapper) Write(p []byte) (n int, err error) {
+	n, err = wc.w.Write(p)
+	wc.c += int64(n)
+	return
+}

cmd/ctr/rpull.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/containerd/accelerated-container-image/pkg/label"
+
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cmd/ctr/commands"
 	ctrcontent "github.com/containerd/containerd/cmd/ctr/commands/content"
@@ -41,6 +43,10 @@ var rpullCommand = cli.Command{
 			Value:  "overlaybd",
 			EnvVar: "CONTAINERD_SNAPSHOTTER",
 		},
+		cli.BoolFlag{
+			Name:  "download-blobs",
+			Usage: "download overlaybd blobs",
+		},
 	),
 	Action: func(context *cli.Context) error {
 		var (
@@ -67,7 +73,7 @@ var rpullCommand = cli.Command{
 			return err
 		}
 
-		if err := rpull(ctx, client, ref, context.String("snapshotter"), config); err != nil {
+		if err := rpull(ctx, client, ref, context.String("snapshotter"), config, context.Bool("download-blobs")); err != nil {
 			return err
 		}
 
@@ -79,7 +85,7 @@ var rpullCommand = cli.Command{
 // rpull pulls image with special labels
 //
 // NOTE: Based on https://github.com/containerd/containerd/blob/v1.5.0-beta.2/cmd/ctr/commands/content/fetch.go#L148.
-func rpull(ctx context.Context, client *containerd.Client, ref string, sn string, config *ctrcontent.FetchConfig) error {
+func rpull(ctx context.Context, client *containerd.Client, ref string, sn string, config *ctrcontent.FetchConfig, download bool) error {
 	ongoing := ctrcontent.NewJobs(ref)
 
 	pctx, stopProgress := context.WithCancel(ctx)
@@ -97,7 +103,10 @@ func rpull(ctx context.Context, client *containerd.Client, ref string, sn string
 		return nil, nil
 	})
 
-	snLabels := map[string]string{"containerd.io/snapshot/image-ref": ref}
+	snLabels := map[string]string{label.TargetImageRef: ref}
+	if download {
+		snLabels[label.DownloadRemoteBlob] = "download"
+	}
 
 	labels := commands.LabelArgs(config.Labels)
 

pkg/label/label.go

@@ -64,13 +64,13 @@ const (
 	// RecordTrace tells snapshotter to record trace
 	RecordTrace = "containerd.io/snapshot/overlaybd/record-trace"
 
-	// RecordTracePath is the the file path to record trace
+	// RecordTracePath is the file path to record trace
 	RecordTracePath = "containerd.io/snapshot/overlaybd/record-trace-path"
 
 	// ZFileConfig is the config of ZFile
 	ZFileConfig = "containerd.io/snapshot/overlaybd/zfile-config"
 
-	// CRIImageRef is thr image-ref from cri
+	// CRIImageRef is the image-ref from cri
 	CRIImageRef = "containerd.io/snapshot/cri.image-ref"
 
 	// FastOCIDigest is the index annotation key for image layer digest
@@ -79,6 +79,9 @@ const (
 	// FastOCIMediaType is the index annotation key for image layer media type
 	FastOCIMediaType = "containerd.io/snapshot/overlaybd/fastoci/target-media-type"
 
+	// DownloadRemoteBlob is a label for download remote blob
+	DownloadRemoteBlob = "containerd.io/snapshot/overlaybd/download-remote-blob"
+
 	RemoteLabel    = "containerd.io/snapshot/remote"
 	RemoteLabelVal = "remote snapshot"
 )

pkg/snapshot/overlay.go

@@ -401,6 +401,29 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 		if err != nil {
 			return nil, err
 		}
+
+		// Download blob
+		downloadBlob := info.Labels[label.DownloadRemoteBlob]
+		if downloadBlob == "download" && stype == storageTypeRemoteBlock {
+			log.G(ctx).Infof("download blob %s", targetRef)
+			if err := o.constructOverlayBDSpec(ctx, key, false); err != nil {
+				return nil, err
+			}
+			rollback = false
+			if err := t.Commit(); err != nil {
+				return nil, err
+			}
+			return []mount.Mount{{
+				Source: o.upperPath(id),
+				Type:   "bind",
+				Options: []string{
+					"rw",
+					"bind",
+				}},
+			}, nil
+		}
+
+		// Normal prepare
 		if _, isFastOCI := info.Labels[label.FastOCIDigest]; isFastOCI {
 			log.G(ctx).Debugf("%s is FastOCI layer", s.ID)
 			if err := o.constructOverlayBDSpec(ctx, key, false); err != nil {
@@ -530,6 +553,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 
 // Prepare creates an active snapshot identified by key descending from the provided parent.
 func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) (_ []mount.Mount, retErr error) {
+	log.G(ctx).Debugf("Prepare (key: %s, parent: %s)", key, parent)
 	start := time.Now()
 	defer func() {
 		if retErr != nil {