Skip to content

Commit

Permalink
bugfix: set device limit by the mountpoint
Browse files Browse the repository at this point in the history 
set device limit by the mountpoint.

Signed-off-by: Rudy Zhang <[email protected]>
  • Loading branch information
@rudyfly
rudyfly committed Aug 24, 2018
1 parent 90f165a commit 8d234c0
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 59 deletions.
10 changes: 10 additions & 0 deletions storage/quota/grpquota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ func (quota *GrpQuotaDriver) EnforceQuota(dir string) (string, error) {
return "", errors.Wrapf(err, "failed to get deivce id for directory: (%s)", dir)
}

// set limit of dir's device in driver
if _, err = setDevLimit(dir, devID); err != nil {
return "", errors.Wrapf(err, "failed to set device limit, dir: (%s), devID: (%d)", dir, devID)
}

quota.lock.Lock()
defer quota.lock.Unlock()

Expand Down Expand Up @@ -227,11 +232,16 @@ func (quota *GrpQuotaDriver) SetDiskQuota(dir string, size string, quotaID uint3
return errors.Errorf("failed to find quota id to set subtree")
}

// transfer limit from kbyte to byte
limit, err := bytefmt.ToKilobytes(size)
if err != nil {
return errors.Wrapf(err, "failed to change size: (%s) to kilobytes", size)
}

if err := checkDevLimit(dir, limit*1024); err != nil {
return err
}

return quota.setQuota(id, limit, mountPoint)
}

Expand Down
58 changes: 3 additions & 55 deletions storage/quota/prjquota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strconv"
"strings"
"sync"
"syscall"

"github.com/alibaba/pouch/pkg/bytefmt"
"github.com/alibaba/pouch/pkg/exec"
Expand Down Expand Up @@ -37,11 +36,6 @@ type PrjQuotaDriver struct {
// value: the mountpoint of the device in the filesystem
mountPoints map[uint64]string

// devLimits saves all the limit of device.
// key: device ID
// value: the storage upper limit size of the device(unit:B)
devLimits map[uint64]uint64

// lastID is used to mark last used quota ID.
// quota ID is allocated increasingly by sequence one by one.
lastID uint32
Expand All @@ -57,7 +51,7 @@ func (quota *PrjQuotaDriver) EnforceQuota(dir string) (string, error) {
}

// set limit of dir's device in driver
if _, err = quota.setDevLimit(dir, devID); err != nil {
if _, err = setDevLimit(dir, devID); err != nil {
return "", errors.Wrapf(err, "failed to set device limit, dir: (%s), devID: (%d)", dir, devID)
}

Expand Down Expand Up @@ -150,13 +144,13 @@ func (quota *PrjQuotaDriver) SetDiskQuota(dir string, size string, quotaID uint3
return errors.Errorf("failed to find quota id to set subtree")
}

// transfer limit from kbyte to byte
limit, err := bytefmt.ToKilobytes(size)
if err != nil {
return errors.Wrapf(err, "failed to change size: (%s) to kilobytes", size)
}

// transfer limit from kbyte to byte
if err := quota.checkDevLimit(dir, limit*1024); err != nil {
if err := checkDevLimit(dir, limit*1024); err != nil {
return errors.Wrapf(err, "failed to check device limit, dir: (%s), limit: (%d)kb", dir, limit)
}

Expand Down Expand Up @@ -314,49 +308,3 @@ func (quota *PrjQuotaDriver) GetNextQuotaID() (uint32, error) {
logrus.Debugf("get next project quota id: %d", id)
return id, nil
}

// setDevLimit sets device storage upper limit in quota driver according to inpur dir.
func (quota *PrjQuotaDriver) setDevLimit(dir string, devID uint64) (uint64, error) {
if limit, exist := quota.devLimits[devID]; exist {
return limit, nil
}

// get storage upper limit of the device which the dir is on.
var stfs syscall.Statfs_t
if err := syscall.Statfs(dir, &stfs); err != nil {
logrus.Errorf("failed to get path: (%s) limit, err: (%v)", dir, err)
return 0, errors.Wrapf(err, "failed to get path: (%s) limit", dir)
}
limit := stfs.Blocks * uint64(stfs.Bsize)

quota.lock.Lock()
quota.devLimits[devID] = limit
quota.lock.Unlock()

logrus.Debugf("SetDevLimit: dir %s limit is %v B", dir, limit)
return limit, nil
}

// checkDevLimit checks if the device on which the input dir lies has already been recorded in driver.
func (quota *PrjQuotaDriver) checkDevLimit(dir string, size uint64) error {
devID, err := system.GetDevID(dir)
if err != nil {
return errors.Wrapf(err, "failed to get device id, dir: (%s)", dir)
}

limit, exist := quota.devLimits[devID]
if !exist {
// if has not recorded, just add (dir, device, limit) to driver.
if limit, err = quota.setDevLimit(dir, devID); err != nil {
return errors.Wrapf(err, "failed to set device limit, dir: (%s), devID: (%d)", dir, devID)
}
}

if limit < size {
return fmt.Errorf("dir %s quota limit %v must be less than %v", dir, size, limit)
}

logrus.Debugf("succeeded in checkDevLimit (dir %s quota limit %v B) with size %v B", dir, limit, size)

return nil
}
104 changes: 100 additions & 4 deletions storage/quota/quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"

"github.com/alibaba/pouch/pkg/exec"
"github.com/alibaba/pouch/pkg/kernel"
"github.com/alibaba/pouch/pkg/system"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
Expand All @@ -26,11 +29,19 @@ const (
procMountFile = "/proc/mounts"
)

var hasQuota bool

var (
hasQuota bool

// GQuotaDriver represents global quota driver.
GQuotaDriver = NewQuotaDriver("")

// devLimits saves all the limit of device.
// key: device ID
// value: the storage upper limit size of the device(unit:B)
devLimits map[uint64]uint64

// the lock for devLimits
lock sync.Mutex
)

// BaseQuota defines the quota operation interface.
Expand Down Expand Up @@ -80,15 +91,13 @@ func NewQuotaDriver(name string) BaseQuota {
quota = &PrjQuotaDriver{
quotaIDs: make(map[uint32]struct{}),
mountPoints: make(map[uint64]string),
devLimits: make(map[uint64]uint64),
}
default:
kernelVersion, err := kernel.GetKernelVersion()
if err == nil && kernelVersion.Kernel >= 4 {
quota = &PrjQuotaDriver{
quotaIDs: make(map[uint32]struct{}),
mountPoints: make(map[uint64]string),
devLimits: make(map[uint64]uint64),
}
} else {
quota = &GrpQuotaDriver{
Expand Down Expand Up @@ -320,3 +329,90 @@ func loadQuotaIDs(repquotaOpt string) (map[uint32]struct{}, uint32, error) {
logrus.Infof("Load repquota ids: %d, list: %v", len(quotaIDs), quotaIDs)
return quotaIDs, minID, nil
}

func getMountpoint(dir string) (string, error) {
var (
mountPoint string
)

output, err := ioutil.ReadFile(procMountFile)
if err != nil {
logrus.Warnf("failed to read file: (%s), err: (%v)", procMountFile, err)
return "", errors.Wrapf(err, "failed to read file: (%s)", procMountFile)
}

// /dev/sdb1 /home/pouch ext4 rw,relatime,prjquota,data=ordered 0 0
for _, line := range strings.Split(string(output), "\n") {
parts := strings.Split(line, " ")
if len(parts) != 6 {
continue
}

if strings.HasPrefix(dir, parts[1]) && len(parts[1]) > len(mountPoint) {
mountPoint = parts[1]
}
}

if mountPoint == "" {
return "", errors.Errorf("failed to get mount point of directory: (%s)", dir)
}

return mountPoint, nil
}

// setDevLimit sets device storage upper limit in quota driver according to inpur dir.
func setDevLimit(dir string, devID uint64) (uint64, error) {
if limit, exist := devLimits[devID]; exist {
return limit, nil
}

mp, err := getMountpoint(dir)
if err != nil {
return 0, errors.Wrapf(err, "failed to set device limit, dir: (%s), devID: (%d)", dir, devID)
}

newDevID, _ := system.GetDevID(mp)
if newDevID != devID {
return 0, errors.Errorf("failed to set device limit, no such device id: (%d), checked id: (%d)",
devID, newDevID)
}

// get storage upper limit of the device which the dir is on.
var stfs syscall.Statfs_t
if err := syscall.Statfs(mp, &stfs); err != nil {
logrus.Errorf("failed to get path: (%s) limit, err: (%v)", mp, err)
return 0, errors.Wrapf(err, "failed to get path: (%s) limit", mp)
}
limit := stfs.Blocks * uint64(stfs.Bsize)

lock.Lock()
devLimits[devID] = limit
lock.Unlock()

logrus.Debugf("SetDevLimit: dir: (%s), mountpoint: (%s), limit: (%v) B", dir, mp, limit)
return limit, nil
}

// checkDevLimit checks if the device on which the input dir lies has already been recorded in driver.
func checkDevLimit(dir string, size uint64) error {
devID, err := system.GetDevID(dir)
if err != nil {
return errors.Wrapf(err, "failed to get device id, dir: (%s)", dir)
}

limit, exist := devLimits[devID]
if !exist {
// if has not recorded, just add (dir, device, limit) to driver.
if limit, err = setDevLimit(dir, devID); err != nil {
return errors.Wrapf(err, "failed to set device limit, dir: (%s), devID: (%d)", dir, devID)
}
}

if limit < size {
return fmt.Errorf("dir %s quota limit %v must be less than %v", dir, size, limit)
}

logrus.Debugf("succeeded in checkDevLimit (dir %s quota limit %v B) with size %v B", dir, limit, size)

return nil
}

0 comments on commit 8d234c0

Please sign in to comment.