feature: add disk quota for container metadata directory

Add disk quota for container metadata directory, use to limit the size of container's log files. Signed-off-by: Rudy Zhang <[email protected]>
AliyunContainerService · Aug 21, 2018 · d09f406 · d09f406
1 parent 318aad0
commit d09f406
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 38 deletions.
diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go
@@ -355,33 +355,9 @@ func (mgr *ContainerManager) Create(ctx context.Context, name string, config *ty
 	// after create options passed to containerd.
 	mgr.setBaseFS(ctx, container, id)
 
-	if err := mgr.Mount(ctx, container); err != nil {
-		return nil, errors.Wrapf(err, "failed to mount container: (%s) rootfs: (%s)", id, container.MountFS)
-	}
-
-	// parse volume config
-	if err := mgr.generateMountPoints(ctx, container); err != nil {
-		if err = mgr.Unmount(ctx, container); err != nil {
-			err = errors.Wrapf(err, "failed to umount container: (%s) rootfs: (%s)", id, container.MountFS)
-		}
-		return nil, errors.Wrap(err, "failed to parse volume argument")
-	}
-
-	// set mount point disk quota
-	if err := mgr.setMountPointDiskQuota(ctx, container); err != nil {
-		if err = mgr.Unmount(ctx, container); err != nil {
-			err = errors.Wrapf(err, "failed to umount container: (%s) rootfs: (%s)", id, container.MountFS)
-		}
-		return nil, errors.Wrap(err, "failed to set mount point disk quota")
-	}
-
-	// set rootfs disk quota
-	if err := mgr.setRootfsQuota(ctx, container); err != nil {
-		logrus.Warnf("failed to set rootfs disk quota, err: %v", err)
-	}
-
-	if err := mgr.Unmount(ctx, container); err != nil {
-		return nil, errors.Wrapf(err, "failed to umount container: (%s) rootfs: (%s)", id, container.MountFS)
+	// init container storage module, such as: set volumes, set diskquota, set /etc/mtab, copy image's data to volume.
+	if err := mgr.initContainerStorage(ctx, container); err != nil {
+		return nil, errors.Wrapf(err, "failed to init container storage, id: (%s)", container.ID)
 	}
 
 	// set network settings
@@ -1193,15 +1169,36 @@ func (mgr *ContainerManager) updateContainerDiskQuota(ctx context.Context, c *Co
 	}
 	// update container rootfs disk quota
 	// TODO: add lock for container?
+	rootfs := ""
 	if c.IsRunningOrPaused() && c.Snapshotter != nil {
 		basefs, ok := c.Snapshotter.Data["MergedDir"]
 		if !ok || basefs == "" {
 			return fmt.Errorf("Container is running, but MergedDir is missing")
 		}
-
-		if err := quota.SetRootfsDiskQuota(basefs, defaultQuota, qid); err != nil {
-			return errors.Wrapf(err, "failed to set container rootfs diskquota")
+		rootfs = basefs
+	} else {
+		if err := mgr.Mount(ctx, c); err != nil {
+			return errors.Wrapf(err, "failed to mount rootfs: (%s)", c.MountFS)
 		}
+		rootfs = c.MountFS
+
+		defer func() {
+			if err := mgr.Unmount(ctx, c); err != nil {
+				logrus.Errorf("failed to umount rootfs: (%s), err: (%v)", c.MountFS, err)
+			}
+		}()
+	}
+	newID, err := quota.SetRootfsDiskQuota(rootfs, defaultQuota, qid)
+	if err != nil {
+		return errors.Wrapf(err, "failed to set container rootfs diskquota")
+	}
+
+	// set container's metadata directory diskquota, for limit the size of container's logs
+	metaDir := mgr.Store.Path(c.ID)
+	err = quota.SetDiskQuota(metaDir, defaultQuota, newID)
+	if err != nil {
+		return errors.Wrapf(err, "failed to set container's log quota, dir: (%s), quota: (%s), quota id: (%d)",
+			metaDir, defaultQuota, newID)
 	}
 
 	return nil

diff --git a/daemon/mgr/container_storage.go b/daemon/mgr/container_storage.go
@@ -461,12 +461,21 @@ func (mgr *ContainerManager) setRootfsQuota(ctx context.Context, c *Container) e
 		return errors.Wrapf(err, "failed to change quota id: (%s) from string to int", qid)
 	}
 
-	err = quota.SetRootfsDiskQuota(c.MountFS, rootfsQuota, uint32(id))
+	// set rootfs quota
+	newID, err := quota.SetRootfsDiskQuota(c.MountFS, rootfsQuota, uint32(id))
 	if err != nil {
 		return errors.Wrapf(err, "failed to set rootfs quota, mountfs: (%s), quota: (%s), quota id: (%d)",
 			c.MountFS, rootfsQuota, id)
 	}
 
+	// set container's metadata directory diskquota, for limit the size of container's logs
+	metaDir := mgr.Store.Path(c.ID)
+	err = quota.SetDiskQuota(metaDir, rootfsQuota, newID)
+	if err != nil {
+		return errors.Wrapf(err, "failed to set container's log quota, dir: (%s), quota: (%s), quota id: (%d)",
+			metaDir, rootfsQuota, newID)
+	}
+
 	return nil
 }
 
@@ -640,6 +649,35 @@ func (mgr *ContainerManager) Unmount(ctx context.Context, c *Container) error {
 	return os.RemoveAll(c.MountFS)
 }
 
+func (mgr *ContainerManager) initContainerStorage(ctx context.Context, c *Container) (err error) {
+	if err = mgr.Mount(ctx, c); err != nil {
+		return errors.Wrapf(err, "failed to mount rootfs: (%s)", c.MountFS)
+	}
+
+	defer func() {
+		if ret := mgr.Unmount(ctx, c); ret != nil {
+			err = errors.Wrapf(err, "failed to umount rootfs: (%s), err: (%v)", c.MountFS, ret)
+		}
+	}()
+
+	// parse volume config
+	if err = mgr.generateMountPoints(ctx, c); err != nil {
+		return errors.Wrap(err, "failed to parse volume argument")
+	}
+
+	// set mount point disk quota
+	if err = mgr.setMountPointDiskQuota(ctx, c); err != nil {
+		return errors.Wrap(err, "failed to set mount point disk quota")
+	}
+
+	// set rootfs disk quota
+	if err = mgr.setRootfsQuota(ctx, c); err != nil {
+		logrus.Warnf("failed to set rootfs disk quota, err: %v", err)
+	}
+
+	return nil
+}
+
 func copyImageContent(source, destination string) error {
 	fi, err := os.Stat(source)
 	if err != nil {

diff --git a/storage/quota/quota.go b/storage/quota/quota.go
@@ -167,33 +167,33 @@ func GetDefaultQuota(quotas map[string]string) string {
 }
 
 // SetRootfsDiskQuota is to set container rootfs dir disk quota.
-func SetRootfsDiskQuota(basefs, size string, quotaID uint32) error {
+func SetRootfsDiskQuota(basefs, size string, quotaID uint32) (uint32, error) {
 	overlayMountInfo, err := getOverlayMountInfo(basefs)
 	if err != nil {
-		return fmt.Errorf("failed to get overlay mount info: %v", err)
+		return 0, fmt.Errorf("failed to get overlay mount info: %v", err)
 	}
 
 	for _, dir := range []string{overlayMountInfo.Upper, overlayMountInfo.Work} {
 		_, err = StartQuotaDriver(dir)
 		if err != nil {
-			return fmt.Errorf("failed to start quota driver: %v", err)
+			return 0, fmt.Errorf("failed to start quota driver: %v", err)
 		}
 
 		quotaID, err = SetSubtree(dir, quotaID)
 		if err != nil {
-			return fmt.Errorf("failed to set subtree: %v", err)
+			return 0, fmt.Errorf("failed to set subtree: %v", err)
 		}
 
 		if err := SetDiskQuota(dir, size, quotaID); err != nil {
-			return fmt.Errorf("failed to set disk quota: %v", err)
+			return 0, fmt.Errorf("failed to set disk quota: %v", err)
 		}
 
 		if err := setQuotaForDir(dir, quotaID); err != nil {
-			return fmt.Errorf("failed to set dir quota: %v", err)
+			return 0, fmt.Errorf("failed to set dir quota: %v", err)
 		}
 	}
 
-	return nil
+	return quotaID, nil
 }
 
 // setQuotaForDir sets file attribute

diff --git a/test/cli_run_volume_test.go b/test/cli_run_volume_test.go
@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"strings"
 
@@ -12,6 +13,7 @@ import (
 
 	"github.com/go-check/check"
 	"github.com/gotestyourself/gotestyourself/icmd"
+	"github.com/pkg/errors"
 )
 
 // PouchRunVolumeSuite is the test suite for run CLI.
@@ -331,3 +333,65 @@ func (suite *PouchRunVolumeSuite) TestRunWithDiskQuota(c *check.C) {
 
 	c.Assert(found, check.Equals, true)
 }
+
+// TestRunWithDiskQuotaForLog tests limit the size of container's log.
+func (suite *PouchRunVolumeSuite) TestRunWithDiskQuotaForLog(c *check.C) {
+	if !environment.IsDiskQuota() {
+		c.Skip("Host does not support disk quota")
+	}
+
+	cname := "TestRunWithDiskQuotaForLog"
+	command.PouchRun("run", "-d", "--disk-quota", "10m",
+		"--name", cname, busyboxImage, "top").Assert(c, icmd.Success)
+
+	defer DelContainerForceMultyTime(c, cname)
+
+	containerMetaDir, err := getContainerMetaDir(cname)
+	c.Assert(err, check.Equals, nil)
+
+	testFile := filepath.Join(containerMetaDir, "diskquota_testfile")
+
+	icmd.RunCommand("groupadd", "quota").Assert(c, icmd.Success)
+	defer icmd.RunCommand("groupdel", "quota").Assert(c, icmd.Success)
+
+	icmd.RunCommand("useradd", "-g", "quota", "quota").Assert(c, icmd.Success)
+	defer icmd.RunCommand("userdel", "quota").Assert(c, icmd.Success)
+
+	expct := icmd.Expected{
+		ExitCode: 1,
+		Err:      "Disk quota exceeded",
+	}
+	err = icmd.RunCommand("dd", "if=/dev/zero", "of="+testFile, "bs=1M", "count=20", "oflag=direct").Compare(expct)
+	c.Assert(err, check.IsNil)
+}
+
+func getContainerMetaDir(name string) (string, error) {
+	ret := command.PouchRun("inspect", name)
+
+	mergedDir := ""
+	for _, line := range strings.Split(ret.Stdout(), "\n") {
+		if strings.Contains(line, "MergedDir") {
+			mergedDir = strings.Split(line, "\"")[3]
+			break
+		}
+	}
+
+	var (
+		graph string
+		cid   string
+	)
+	if mergedDir == "" {
+		return "", errors.Errorf("failed to get container metadata directory")
+	}
+
+	parts := strings.Split(mergedDir, "/")
+	for i, part := range parts {
+		if part == "containerd" {
+			graph = "/" + filepath.Join(parts[:i]...)
+			cid = parts[i+4]
+			break
+		}
+	}
+
+	return filepath.Join(graph, "containers", cid), nil
+}