Skip to content

Commit

Permalink
refactor: make all quota implementation logic clear
Browse files Browse the repository at this point in the history
Signed-off-by: Allen Sun <[email protected]>
  • Loading branch information
allencloud committed Jul 27, 2018
1 parent e42ceba commit 55b50ef
Show file tree
Hide file tree
Showing 12 changed files with 610 additions and 378 deletions.
12 changes: 10 additions & 2 deletions apis/swagger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1957,7 +1957,12 @@ definitions:
description: "Initial script executed in container. The script will be executed before entrypoint or command"
DiskQuota:
type: "object"
description: "Set disk quota for container"
description: |
Set disk quota for container.
Key is the dir in container.
Value is disk quota size for the dir.
/ means rootfs dir in container.
.* includes rootfs dir and all volume dir.
x-nullable: true
additionalProperties:
type: "string"
Expand All @@ -1968,7 +1973,10 @@ definitions:
type: "string"
QuotaID:
type: "string"
description: "set disk quota by specified quota id, if id < 0, it means pouchd alloc a unique quota id"
description: |
Set disk quota by specified quota id.
If QuotaID <= 0, it means pouchd should allocate a unique quota id by sequence automatically.
By default, a quota ID is mapped to only one container. And one quota ID can include several mountpoint.
NetPriority:
description: "net priority."
type: "integer"
Expand Down
12 changes: 10 additions & 2 deletions apis/types/container_config.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

76 changes: 24 additions & 52 deletions daemon/mgr/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"os/exec"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -1068,6 +1066,8 @@ func (mgr *ContainerManager) Remove(ctx context.Context, name string, options *t
return nil
}

// updateContainerDiskQuota updates the disk quota config of container.
// This action will take effect on the host immediately.
func (mgr *ContainerManager) updateContainerDiskQuota(ctx context.Context, c *Container, diskQuota map[string]string) error {
if diskQuota == nil {
return nil
Expand All @@ -1084,27 +1084,14 @@ func (mgr *ContainerManager) updateContainerDiskQuota(ctx context.Context, c *Co
return errors.Wrapf(err, "failed to set mount point disk quota")
}

c.Lock()
var qid uint32
if c.Config.QuotaID != "" {
id, err := strconv.Atoi(c.Config.QuotaID)
if err != nil {
return errors.Wrapf(err, "failed to convert QuotaID %s", c.Config.QuotaID)
}

qid = uint32(id)
if id < 0 {
// QuotaID is < 0, it means pouchd alloc a unique quota id.
qid, err = quota.GetNextQuatoID()
if err != nil {
return errors.Wrap(err, "failed to get next quota id")
}

// update QuotaID
c.Config.QuotaID = strconv.Itoa(int(qid))
}
var (
qid uint32
err error
)
c.Config.QuotaID, qid, err = getQuotaID(c.Config.QuotaID)
if err != nil {
return err
}
c.Unlock()

// get rootfs quota
defaultQuota := quota.GetDefaultQuota(c.Config.DiskQuota)
Expand Down Expand Up @@ -2229,37 +2216,27 @@ func (mgr *ContainerManager) populateVolumes(ctx context.Context, c *Container)
return nil
}

// setMountPointDiskQuota sets disk quota for mountpoints.
// These mountpoints includes the following parts:
// * bind mount via -v option
// * daemon created volume
func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Container) error {
if c.Config.DiskQuota == nil {
if len(c.Config.DiskQuota) = 0 {
if c.Config.QuotaID != "" && c.Config.QuotaID != "0" {
return fmt.Errorf("invalid argument, set quota-id without disk-quota")
return fmt.Errorf("user cannot set non-zero quota ID if no disk quota is set")
}
return nil
}

var (
qid uint32
setQuotaID bool
err error
)

if c.Config.QuotaID != "" {
id, err := strconv.Atoi(c.Config.QuotaID)
if err != nil {
return errors.Wrapf(err, "invalid argument, QuotaID: %s", c.Config.QuotaID)
}

// if QuotaID is < 0, it means pouchd alloc a unique quota id.
if id < 0 {
qid, err = quota.GetNextQuatoID()
if err != nil {
return errors.Wrap(err, "failed to get next quota id")
}

// update QuotaID
c.Config.QuotaID = strconv.Itoa(int(qid))
} else {
qid = uint32(id)
}
c.Config.QuotaID, qid, err = getQuotaID(c.Config.QuotaID)
if err != nil {
return err
}

if qid > 0 {
Expand All @@ -2274,11 +2251,7 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont
}

// parse diskquota regexe
var res []*quota.RegExp
for path, size := range quotas {
re := regexp.MustCompile(path)
res = append(res, &quota.RegExp{Pattern: re, Path: path, Size: size})
}
diskRegExps := parseDiskQuota(quotas)

for _, mp := range c.Mounts {
// skip volume mount or replace mode mount
Expand All @@ -2290,12 +2263,12 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont
if mp.Name != "" {
v, err := mgr.VolumeMgr.Get(ctx, mp.Name)
if err != nil {
logrus.Warnf("failed to get volume: %s", mp.Name)
logrus.Warnf("failed to get volume %s when updating disk quota of container %s: %v", mp.Name, c.ID, err)
continue
}

if v.Size() != "" {
logrus.Debugf("skip volume: %s with size", mp.Name)
logrus.Debugf("skip volume %s with size", mp.Name)
continue
}
}
Expand All @@ -2307,7 +2280,7 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont
}

matched := false
for _, re := range res {
for _, re := range diskRegExps {
findStr := re.Pattern.FindString(mp.Destination)
if findStr == mp.Destination {
quotas[mp.Destination] = re.Size
Expand All @@ -2324,8 +2297,7 @@ func (mgr *ContainerManager) setMountPointDiskQuota(ctx context.Context, c *Cont
} else {
size = defaultQuota
}
err := quota.SetDiskQuota(mp.Source, size, qid)
if err != nil {
if err := quota.SetDiskQuota(mp.Source, size, qid); err != nil {
return err
}
}
Expand Down
110 changes: 110 additions & 0 deletions daemon/mgr/container_quota.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package mgr

import (
"regexp"
"strconv"

"github.com/alibaba/pouch/storage/quota"

"github.com/pkg/errors"
)

// GetQuotaID generate a valid quota ID according to the input one.
// If quotaID < 0, allocate a unique ID via quota driver;
// If quotaID = 0, do not set disk quota;
// Otherwise, return the input one.
//
// first return parameter is quota ID in string;
// second is quota ID in uint32;
// third is an error instance.
func getQuotaID(quotaID string) (string, uint32, error) {
var (
id int
qid uint32
err error
)

if quotaID == "" {
return quotaID, uint32(0), nil
}

id, err = strconv.Atoi(quotaID)
if err != nil {
return quotaID, uint32(0), errors.Wrapf(err, "invalid argument, quotaID: %s", quotaID)
}

qid = uint32(id)
if id > 0 {
return quotaID, uint32(0), nil
}

// if QuotaID is <= 0, it means pouchd alloc a unique quota id.
qid, err = quota.GetNextQuotaID()
if err != nil {
return quotaID, qid, errors.Wrap(err, "failed to get next quota id")
}
quotaID = strconv.Itoa(int(qid))

return quotaID, qid, nil
}

func parseDiskQuota(diskQuotas map[string]string) []*quota.RegExp {
regExps := make(*quota.RegExp, 0)
for path, size := range quotas {
re := regexp.MustCompile(path)
regExps = append(res, &quota.RegExp{Pattern: re, Path: path, Size: size})
}
return regExps
}

func (mgr *ContainerManager) setDiskQuotaForContainer(mounts []*types.MountPoint, quotas [string]string) error{
for _, mp := range mounts {
// skip volume mount or replace mode mount
if mp.Replace != "" || mp.Source == "" || mp.Destination == "" {
logrus.Debugf("skip volume mount or replace mode mount")
continue
}

if mp.Name != "" {
v, err := mgr.VolumeMgr.Get(ctx, mp.Name)
if err != nil {
logrus.Warnf("failed to get volume %s when updating disk quota of container %s: %v", mp.Name, c.ID, err)
continue
}

if v.Size() != "" {
logrus.Debugf("skip volume %s with size", mp.Name)
continue
}
}

// skip non-directory path.
if fd, err := os.Stat(mp.Source); err != nil || !fd.IsDir() {
logrus.Debugf("skip non-directory path: %s", mp.Source)
continue
}

matched := false
for _, re := range diskRegExps {
findStr := re.Pattern.FindString(mp.Destination)
if findStr == mp.Destination {
quotas[mp.Destination] = re.Size
matched = true
if re.Path != ".*" {
break
}
}
}

size := ""
if matched && !setQuotaID {
size = quotas[mp.Destination]
} else {
size = defaultQuota
}
if err := quota.SetDiskQuota(mp.Source, size, qid); err != nil {
return err
}
}
return nil
}
5 changes: 3 additions & 2 deletions daemon/mgr/spec_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"strconv"
"strings"

"github.com/alibaba/pouch/pkg/system"
"github.com/alibaba/pouch/storage/quota"

specs "github.com/opencontainers/runtime-spec/specs-go"
Expand Down Expand Up @@ -90,7 +91,7 @@ func setMountTab(ctx context.Context, c *Container, spec *SpecWrapper) error {

// set rootfs mount tab
context := "/ / ext4 rw 0 0\n"
if rootID, e := quota.GetDevID(c.BaseFS); e == nil {
if rootID, e := system.GetDevID(c.BaseFS); e == nil {
_, _, rootFsType := quota.CheckMountpoint(rootID)
if len(rootFsType) > 0 {
context = fmt.Sprintf("/ / %s rw 0 0\n", rootFsType)
Expand All @@ -110,7 +111,7 @@ func setMountTab(ctx context.Context, c *Container, spec *SpecWrapper) error {
}

tempLine := fmt.Sprintf("/dev/v%02dd %s ext4 rw 0 0\n", i, m.Destination)
if tmpID, e := quota.GetDevID(m.Source); e == nil {
if tmpID, e := system.GetDevID(m.Source); e == nil {
_, _, fsType := quota.CheckMountpoint(tmpID)
if len(fsType) > 0 {
tempLine = fmt.Sprintf("/dev/v%02dd %s %s rw 0 0\n", i, m.Destination, fsType)
Expand Down
19 changes: 19 additions & 0 deletions pkg/system/device.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// +build linux

package system

import (
"syscall"

"github.com/sirupsen/logrus"
)

// GetDevID returns device id via syscall according to the input directory.
func GetDevID(dir string) (uint64, error) {
var st syscall.Stat_t
if err := syscall.Stat(dir, &st); err != nil {
logrus.Warnf("failed to get device id of dir %s: %v", dir, err)
return 0, err
}
return st.Dev, nil
}
2 changes: 2 additions & 0 deletions pkg/system/sysinfo.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// +build linux

package system

import (
Expand Down
Loading

0 comments on commit 55b50ef

Please sign in to comment.