From 585f307b38faf537ffe9d46084e5d4c754c45af0 Mon Sep 17 00:00:00 2001 From: Yifan Yuan Date: Tue, 24 Dec 2024 12:23:25 +0800 Subject: [PATCH] [feat.] AsyncRemove support Enable 'asyncRemove' can avoid taking a long time to remove snDir Signed-off-by: Yifan Yuan --- cmd/overlaybd-snapshotter/main.go | 4 +- docs/PROMETHEUS.md | 1 + pkg/snapshot/overlay.go | 90 +++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/cmd/overlaybd-snapshotter/main.go b/cmd/overlaybd-snapshotter/main.go index 937517e2..9c4ac4f0 100644 --- a/cmd/overlaybd-snapshotter/main.go +++ b/cmd/overlaybd-snapshotter/main.go @@ -51,8 +51,8 @@ func parseConfig(fpath string) error { if err := json.Unmarshal(data, pconfig); err != nil { return errors.Wrapf(err, "failed to parse plugin config from %s", string(data)) } - logrus.Infof("snapshotter commitID: %s, rwMode: %s, autoRemove: %v, writableLayerType: %s", - commitID, pconfig.RwMode, pconfig.AutoRemoveDev, pconfig.WritableLayerType) + logrus.Infof("snapshotter commitID: %s, rwMode: %s, autoRemove: %v, writableLayerType: %s, asyncRemoveSnapshot: %v", + commitID, pconfig.RwMode, pconfig.AutoRemoveDev, pconfig.WritableLayerType, pconfig.AsyncRemove) return nil } diff --git a/docs/PROMETHEUS.md b/docs/PROMETHEUS.md index 061df982..0008dc91 100644 --- a/docs/PROMETHEUS.md +++ b/docs/PROMETHEUS.md @@ -9,6 +9,7 @@ To configure overlaybd as a Prometheus target, you need to specify `uriPrefix` a ```json { "root": "/var/lib/containerd/io.containerd.snapshotter.v1.overlaybd", + "asyncRemove": false, "address": "/run/overlaybd-snapshotter/overlaybd.sock", "verbose": "info", "rwMode": "overlayfs", diff --git a/pkg/snapshot/overlay.go b/pkg/snapshot/overlay.go index 5e30f911..b1912bec 100644 --- a/pkg/snapshot/overlay.go +++ b/pkg/snapshot/overlay.go @@ -85,6 +85,7 @@ type Registry struct { } type BootConfig struct { + AsyncRemove bool `json:"asyncRemove"` Address string `json:"address"` Root string `json:"root"` LogLevel string `json:"verbose"` @@ -102,6 +103,7 @@ type BootConfig struct { func DefaultBootConfig() *BootConfig { return &BootConfig{ + AsyncRemove: false, LogLevel: "info", RwMode: "overlayfs", LogReportCaller: false, @@ -142,6 +144,15 @@ var defaultConfig = SnapshotterConfig{ // Opt is an option to configure the snapshotter type Opt func(config *SnapshotterConfig) error +// AsynchronousRemove defers removal of filesystem content until +// the Cleanup method is called. Removals will make the snapshot +// referred to by the key unavailable and make the key immediately +// available for re-use. +func AsynchronousRemove(config *BootConfig) error { + config.AsyncRemove = true + return nil +} + // snapshotter is implementation of github.com/containerd/containerd/snapshots.Snapshotter. // // It is a snapshotter plugin. The layout of root dir is organized: @@ -189,6 +200,7 @@ type snapshotter struct { tenant int locker *locker.Locker turboFsType []string + asyncRemove bool quotaDriver *diskquota.PrjQuotaDriver quotaSize string @@ -256,6 +268,7 @@ func NewSnapshotter(bootConfig *BootConfig, opts ...Opt) (snapshots.Snapshotter, quotaDriver: &diskquota.PrjQuotaDriver{ QuotaIDs: make(map[uint32]struct{}), }, + asyncRemove: bootConfig.AsyncRemove, }, nil } @@ -910,6 +923,63 @@ func (o *snapshotter) commit(ctx context.Context, name, key string, opts ...snap return id, info, nil } +func (o *snapshotter) cleanupDirectories(ctx context.Context) ([]string, error) { + // Get a write transaction to ensure no other write transaction can be entered + // while the cleanup is scanning. + ctx, t, err := o.ms.TransactionContext(ctx, true) + if err != nil { + return nil, err + } + + defer t.Rollback() + return o.getCleanupDirectories(ctx, t) +} +func (o *snapshotter) getCleanupDirectories(ctx context.Context, t storage.Transactor) ([]string, error) { + ids, err := storage.IDMap(ctx) + if err != nil { + return nil, err + } + + snapshotDir := filepath.Join(o.root, "snapshots") + fd, err := os.Open(snapshotDir) + if err != nil { + return nil, err + } + defer fd.Close() + + dirs, err := fd.Readdirnames(0) + if err != nil { + return nil, err + } + + cleanup := []string{} + for _, d := range dirs { + if _, ok := ids[d]; ok { + continue + } + + cleanup = append(cleanup, filepath.Join(snapshotDir, d)) + } + + return cleanup, nil +} + +// Cleanup cleans up disk resources from removed or abandoned snapshots// Cleanup cleans up disk resources from removed or abandoned snapshots +func (o *snapshotter) Cleanup(ctx context.Context) error { + cleanup, err := o.cleanupDirectories(ctx) + if err != nil { + return err + } + + for _, dir := range cleanup { + if err := os.RemoveAll(dir); err != nil { + log.G(ctx).WithError(err).WithField("path", dir).Warn("failed to remove directory") + } + } + + return nil +} + // Remove abandons the snapshot identified by key. The snapshot will // immediately become unavailable and unrecoverable. func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { @@ -988,9 +1058,23 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { if err != nil { return errors.Wrap(err, "failed to remove") } - - if err := os.RemoveAll(o.snPath(id)); err != nil && !os.IsNotExist(err) { - return err + if !o.asyncRemove { + var removals []string + removals, err = o.getCleanupDirectories(ctx, t) + if err != nil { + return errors.Wrap(err, "unable to get directories for removal") + } + defer func() { + if err == nil { + for _, dir := range removals { + if err := os.RemoveAll(dir); err != nil { + log.G(ctx).WithError(err).WithField("path", dir).Warn("failed to remove directory") + } + } + } + }() + } else { + log.G(ctx).Info("asyncRemove enabled, remove snapshots in Cleanup() method.") } rollback = false