Merge pull request #239 from liulanzheng/main

fixes for usage and seal

pkg/snapshot/overlay.go

@@ -280,13 +280,18 @@ func (o *snapshotter) Usage(ctx context.Context, key string) (_ snapshots.Usage,
 		return snapshots.Usage{}, err
 	}
 
-	if info.Kind == snapshots.KindActive {
-		upperPath := o.upperPath(id)
-		du, err := fs.DiskUsage(ctx, upperPath)
-		if err != nil {
-			return snapshots.Usage{}, err
+	stype, err := o.identifySnapshotStorageType(ctx, id, info)
+	if err != nil {
+		return snapshots.Usage{}, err
+	}
+
+	switch info.Kind {
+	case snapshots.KindActive:
+		return o.diskUsageWithBlock(ctx, id, stype)
+	case snapshots.KindCommitted:
+		if stype == storageTypeRemoteBlock || stype == storageTypeLocalBlock {
+			return o.diskUsageWithBlock(ctx, id, stype)
 		}
-		usage = snapshots.Usage(du)
 	}
 	return usage, nil
 }

pkg/snapshot/storage.go

@@ -40,6 +40,7 @@ import (
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/storage"
 	"github.com/containerd/continuity"
+	"github.com/containerd/continuity/fs"
 	"github.com/moby/sys/mountinfo"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -703,3 +704,20 @@ func lookup(dir string) error {
 func isGzipLayerType(mediaType string) bool {
 	return mediaType == specs.MediaTypeImageLayerGzip || mediaType == images.MediaTypeDockerSchema2LayerGzip
 }
+
+func (o *snapshotter) diskUsageWithBlock(ctx context.Context, id string, stype storageType) (snapshots.Usage, error) {
+	usage := snapshots.Usage{}
+	du, err := fs.DiskUsage(ctx, o.upperPath(id))
+	if err != nil {
+		return snapshots.Usage{}, err
+	}
+	usage = snapshots.Usage(du)
+	if stype == storageTypeRemoteBlock || stype == storageTypeLocalBlock {
+		du, err := utils.DiskUsageWithoutMountpoint(ctx, o.blockPath(id))
+		if err != nil {
+			return snapshots.Usage{}, err
+		}
+		usage.Add(snapshots.Usage(du))
+	}
+	return usage, nil
+}

pkg/utils/cmd.go

@@ -64,7 +64,11 @@ func Seal(ctx context.Context, dir, toDir string, opts ...string) error {
 	if err != nil {
 		return errors.Wrapf(err, "failed to seal writable overlaybd: %s", out)
 	}
-	return os.Rename(path.Join(dir, dataFile), path.Join(toDir, sealedFile))
+	if err := os.Rename(path.Join(dir, dataFile), path.Join(toDir, sealedFile)); err != nil {
+		return errors.Wrapf(err, "failed to rename sealed overlaybd file")
+	}
+	os.RemoveAll(path.Join(dir, idxFile))
+	return nil
 }
 
 func Commit(ctx context.Context, dir, toDir string, sealed bool, opts ...string) error {

pkg/utils/du_unix.go

@@ -0,0 +1,81 @@
+/*
+   Copyright The Accelerated Container Image Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Based on https://github.com/containerd/continuity/blob/main/fs/du_unix.go
+// Used to calculate the usage of the block dir, excluding the block/mountpoint dir.
+
+package utils
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/containerd/continuity/fs"
+)
+
+const blocksUnitSize = 512
+
+type inode struct {
+	dev, ino uint64
+}
+
+func newInode(stat *syscall.Stat_t) inode {
+	return inode{
+		dev: uint64(stat.Dev), //nolint: unconvert // dev is uint32 on darwin/bsd, uint64 on linux/solaris/freebsd
+		ino: uint64(stat.Ino), //nolint: unconvert // ino is uint32 on bsd, uint64 on darwin/linux/solaris/freebsd
+	}
+}
+
+func DiskUsageWithoutMountpoint(ctx context.Context, roots ...string) (fs.Usage, error) {
+	var (
+		size   int64
+		inodes = map[inode]struct{}{} // expensive!
+	)
+
+	for _, root := range roots {
+		if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+			if fi.Name() == "mountpoint" {
+				return filepath.SkipDir
+			}
+			if err != nil {
+				return err
+			}
+
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			default:
+			}
+			stat := fi.Sys().(*syscall.Stat_t)
+			inoKey := newInode(stat)
+			if _, ok := inodes[inoKey]; !ok {
+				inodes[inoKey] = struct{}{}
+				size += stat.Blocks * blocksUnitSize
+			}
+
+			return nil
+		}); err != nil {
+			return fs.Usage{}, err
+		}
+	}
+
+	return fs.Usage{
+		Inodes: int64(len(inodes)),
+		Size:   size,
+	}, nil
+}