diff --git a/go.mod b/go.mod index 016b3613..0bcb88da 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,11 @@ require ( github.com/containerd/containerd v1.6.26 github.com/containerd/continuity v0.4.3 github.com/containerd/go-cni v1.1.9 + github.com/containerd/log v0.1.0 github.com/data-accelerator/zdfs v0.1.4 github.com/go-sql-driver/mysql v1.6.0 + github.com/google/fscrypt v0.3.5 + github.com/jessevdk/go-flags v1.5.0 github.com/moby/locker v1.0.1 github.com/moby/sys/mountinfo v0.6.2 github.com/opencontainers/go-digest v1.0.0 @@ -17,7 +20,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.1.3 - github.com/urfave/cli v1.22.12 + github.com/urfave/cli v1.22.14 golang.org/x/sync v0.4.0 golang.org/x/sys v0.18.0 google.golang.org/grpc v1.58.3 @@ -34,7 +37,6 @@ require ( github.com/containerd/console v1.0.3 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/go-runc v1.0.0 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.3 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/containernetworking/cni v1.1.2 // indirect @@ -69,10 +71,10 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/mod v0.11.0 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.10.0 // indirect + golang.org/x/tools v0.13.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d // indirect google.golang.org/protobuf v1.33.0 // indirect gotest.tools/v3 v3.5.0 // indirect diff --git a/go.sum b/go.sum index 68033775..a05e8a5a 100644 --- a/go.sum +++ b/go.sum @@ -33,7 +33,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -194,6 +194,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/fscrypt v0.3.5 h1:RunYtVg2Z79hfh1W1ZP1k7TaSMYkbELUyMMzAmikyTc= +github.com/google/fscrypt v0.3.5/go.mod h1:HyY8Z/kUPrnIKAwuhjrn2tSTM5/s9zfRRTqRMG0mHks= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -253,6 +255,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -426,12 +430,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= -github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -460,6 +465,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -490,8 +496,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -599,6 +605,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -672,8 +679,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/label/label.go b/pkg/label/label.go index 58e11485..7df5fc03 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -97,6 +97,9 @@ const ( LayerToTurboOCI = "containerd.io/snapshot/overlaybd/convert2turbo-oci" SnapshotType = "containerd.io/snapshot/type" + + // RootfsQuotaLabel sets container rootfs diskquota + RootfsQuotaLabel = "containerd.io/snapshot/disk_quota" ) // used in filterAnnotationsForSave (https://github.com/moby/buildkit/blob/v0.11/cache/refs.go#L882) diff --git a/pkg/snapshot/diskquota/prjquota.go b/pkg/snapshot/diskquota/prjquota.go new file mode 100644 index 00000000..158f321d --- /dev/null +++ b/pkg/snapshot/diskquota/prjquota.go @@ -0,0 +1,235 @@ +//go:build linux +// +build linux + +/* + Copyright The Accelerated Container Image Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package diskquota + +import ( + "fmt" + "math" + "path" + "strconv" + "strings" + "sync" + + "github.com/docker/go-units" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/containerd/containerd/mount" +) + +const ( + // QuotaMinID represents the minimum quota id. + // The value is unit32(2^24). + QuotaMinID = uint32(16777216) + + // QuotaMaxID represents the maximum quota id. + QuotaMaxID = uint32(200000000) +) + +func safeConvertToUInt32(strVal string) (uint32, error) { + intVal, err := strconv.Atoi(strVal) + if err != nil { + return 0, fmt.Errorf("failed to parse integer: %w", err) + } + + // Check if the value is within the uint32 range and is non-negative. + if intVal < 0 || intVal > math.MaxUint32 { + return 0, fmt.Errorf("value %d is out of range for uint32", intVal) + } + + return uint32(intVal), nil +} + +// SetDiskQuotaBytes set dir project quota to the quotaId +func SetDiskQuotaBytes(dir string, limit int64, quotaID uint32) error { + driver := &PrjQuotaDriver{} + mountPoint, hasQuota, err := driver.CheckMountpoint(dir) + if err != nil { + return err + } + if !hasQuota { + // no need to remount option prjquota for mountpoint + return fmt.Errorf("mountpoint: (%s) not enable prjquota", mountPoint) + } + + if err := checkDevLimit(mountPoint, uint64(limit)); err != nil { + return errors.Wrapf(err, "failed to check device limit, dir: (%s), limit: (%d)kb", dir, limit) + } + + err = driver.SetFileAttr(dir, quotaID) + if err != nil { + return errors.Wrapf(err, "failed to set subtree, dir: (%s), quota id: (%d)", dir, quotaID) + } + + return driver.setQuota(quotaID, uint64(limit/1024), mountPoint) +} + +// PrjQuotaDriver represents project quota driver. +type PrjQuotaDriver struct { + lock sync.Mutex + + // quotaIDs saves all of quota ids. + // key: quota ID which means this ID is used in the global scope. + // value: stuct{} + QuotaIDs map[uint32]struct{} + + // lastID is used to mark last used quota ID. + // quota ID is allocated increasingly by sequence one by one. + LastID uint32 +} + +// SetDiskQuota uses the following two parameters to set disk quota for a directory. +// * quota size: a byte size of requested quota. +// * quota ID: an ID represent quota attr which is used in the global scope. +func (quota *PrjQuotaDriver) SetDiskQuota(dir string, size string, quotaID uint32) error { + mountPoint, hasQuota, err := quota.CheckMountpoint(dir) + if err != nil { + return err + } + if !hasQuota { + // no need to remount option prjquota for mountpoint + return fmt.Errorf("mountpoint: (%s) not enable prjquota", mountPoint) + } + + limit, err := units.RAMInBytes(size) + if err != nil { + return errors.Wrapf(err, "failed to change size: (%s) to kilobytes", size) + } + + if err := checkDevLimit(mountPoint, uint64(limit)); err != nil { + return errors.Wrapf(err, "failed to check device limit, dir: (%s), limit: (%d)kb", dir, limit) + } + + err = quota.SetFileAttr(dir, quotaID) + if err != nil { + return errors.Wrapf(err, "failed to set subtree, dir: (%s), quota id: (%d)", dir, quotaID) + } + + return quota.setQuota(quotaID, uint64(limit/1024), mountPoint) +} + +func (quota *PrjQuotaDriver) CheckMountpoint(dir string) (string, bool, error) { + mountInfo, err := mount.Lookup(dir) + if err != nil { + return "", false, errors.Wrapf(err, "failed to get mount info, dir(%s)", dir) + } + if strings.Contains(mountInfo.VFSOptions, "prjquota") { + return mountInfo.Mountpoint, true, nil + } + return mountInfo.Mountpoint, false, nil +} + +// setQuota uses system tool "setquota" to set project quota for binding of limit and mountpoint and quotaID. +// * quotaID: quota ID which means this ID is used in the global scope. +// * blockLimit: block limit number for mountpoint. +// * mountPoint: the mountpoint of the device in the filesystem +// ext4: setquota -P qid $softlimit $hardlimit $softinode $hardinode mountpoint +func (quota *PrjQuotaDriver) setQuota(quotaID uint32, blockLimit uint64, mountPoint string) error { + quotaIDStr := strconv.FormatUint(uint64(quotaID), 10) + blockLimitStr := strconv.FormatUint(blockLimit, 10) + + // ext4 set project quota limit + // logrus.Infof("setquota -P %s 0 %s 0 0 %s", quotaIDStr, blockLimitStr, mountPoint) + stdout, stderr, err := ExecSync("setquota", "-P", quotaIDStr, "0", blockLimitStr, "0", "0", mountPoint) + if err != nil { + return errors.Wrapf(err, "failed to set quota, mountpoint: (%s), quota id: (%d), quota: (%d kbytes), stdout: (%s), stderr: (%s)", + mountPoint, quotaID, blockLimit, stdout, stderr) + } + return nil +} + +// GetQuotaIDInFileAttr gets attributes of the file which is in the inode. +// The returned result is quota ID. +// return 0 if failure happens, since quota ID must be positive. +// execution command: `lsattr -p $dir` +func (quota *PrjQuotaDriver) GetQuotaIDInFileAttr(dir string) uint32 { + parent := path.Dir(dir) + + stdout, _, err := ExecSync("lsattr", "-p", parent) + if err != nil { + // failure, then return invalid value 0 for quota ID. + return 0 + } + + // example output: + // 16777256 --------------e---P ./exampleDir + lines := strings.Split(stdout, "\n") + for _, line := range lines { + parts := strings.Split(line, " ") + if len(parts) > 2 && parts[2] == dir { + // find the corresponding quota ID, return directly. + qid, _ := safeConvertToUInt32(parts[0]) + return qid + } + } + + return 0 +} + +// GetNextQuotaID returns the next available quota id. +func (quota *PrjQuotaDriver) GetNextQuotaID() (quotaID uint32, err error) { + quota.lock.Lock() + defer quota.lock.Unlock() + + if quota.LastID == 0 { + quota.QuotaIDs, quota.LastID, err = loadQuotaIDs("-Pan") + if err != nil { + return 0, errors.Wrap(err, "failed to load quota list") + } + } + id := quota.LastID + for { + if id < QuotaMinID { + id = QuotaMinID + } + id++ + if _, ok := quota.QuotaIDs[id]; !ok { + if id <= QuotaMaxID { + break + } + logrus.Infof("reach the maximum, try to reuse quotaID") + quota.QuotaIDs, quota.LastID, err = loadQuotaIDs("-Pan") + if err != nil { + return 0, errors.Wrap(err, "failed to load quota list") + } + id = quota.LastID + } + } + quota.QuotaIDs[id] = struct{}{} + quota.LastID = id + + return id, nil +} + +// SetFileAttr set the file attr. +// ext4: chattr -p quotaid +P $DIR +func (quota *PrjQuotaDriver) SetFileAttr(dir string, quotaID uint32) error { + strID := strconv.FormatUint(uint64(quotaID), 10) + + // ext4 use chattr to change project id + stdout, stderr, err := ExecSync("chattr", "-p", strID, "+P", dir) + if err != nil { + return errors.Wrapf(err, "failed to set file(%s) quota id(%s), stdout: (%s), stderr: (%s)", + dir, strID, stdout, stderr) + } + logrus.Debugf("set quota id (%s) to file (%s) attr", strID, dir) + + return nil +} diff --git a/pkg/snapshot/diskquota/quota_utils.go b/pkg/snapshot/diskquota/quota_utils.go new file mode 100644 index 00000000..4b6da1fa --- /dev/null +++ b/pkg/snapshot/diskquota/quota_utils.go @@ -0,0 +1,124 @@ +//go:build linux +// +build linux + +/* + Copyright The Accelerated Container Image Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package diskquota + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + "syscall" + + "github.com/pkg/errors" +) + +// CheckRegularFile is used to check the file is regular file or directory. +func CheckRegularFile(file string) (bool, error) { + fd, err := os.Lstat(file) + if err != nil { + return false, err + } + + if fd.Mode()&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) == 0 { + return true, nil + } + + return false, nil +} + +// loadQuotaIDs loads quota IDs for quota driver from reqquota execution result. +// This function utils `repquota` which summarizes quotas for a filesystem. +// see http://man7.org/linux/man-pages/man8/repquota.8.html +// +// $ repquota -Pan +// Project used soft hard grace used soft hard grace +// ---------------------------------------------------------------------- +// #0 -- 220 0 0 25 0 0 +// #123 -- 4 0 88589934592 1 0 0 +// #8888 -- 8 0 0 2 0 0 +func loadQuotaIDs(repquotaOpt string) (map[uint32]struct{}, uint32, error) { + quotaIDs := make(map[uint32]struct{}) + + minID := QuotaMinID + output, stderr, err := ExecSync("repquota", repquotaOpt) + if err != nil { + return nil, 0, errors.Wrapf(err, "failed to execute [repquota %s], stdout: (%s), stderr: (%s)", + repquotaOpt, output, stderr) + } + + lines := strings.Split(output, "\n") + for _, line := range lines { + if len(line) == 0 || line[0] != '#' { + continue + } + // find all lines with prefix '#' + parts := strings.Split(line, " ") + // part[0] is "#123456" + if len(parts[0]) <= 1 { + continue + } + + quotaID, err := safeConvertToUInt32(parts[0][1:]) + if err == nil && quotaID > QuotaMinID { + quotaIDs[quotaID] = struct{}{} + if quotaID > minID { + minID = quotaID + } + } + } + return quotaIDs, minID, nil +} + +// getDevLimit returns the device storage upper limit. +func getDevLimit(mountPoint string) (uint64, error) { + // get storage upper limit of the device which the dir is on. + var stfs syscall.Statfs_t + if err := syscall.Statfs(mountPoint, &stfs); err != nil { + return 0, errors.Wrapf(err, "failed to get path(%s) limit", mountPoint) + } + return stfs.Blocks * uint64(stfs.Bsize), nil +} + +// checkDevLimit checks if the device on which the input dir lies has already been recorded in driver. +func checkDevLimit(mountPoint string, size uint64) error { + limit, err := getDevLimit(mountPoint) + if err != nil { + return errors.Wrapf(err, "failed to get device(%s) limit", mountPoint) + } + + if limit < size { + return fmt.Errorf("dir %s quota limit %v must be less than %v", mountPoint, size, limit) + } + return nil +} + +func ExecSync(bin string, args ...string) (std, serr string, err error) { + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd := exec.Command(bin, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { + return stdout.String(), stderr.String(), err + } + return stdout.String(), stderr.String(), nil +} diff --git a/pkg/snapshot/overlay.go b/pkg/snapshot/overlay.go index f43998ad..2db87cc0 100644 --- a/pkg/snapshot/overlay.go +++ b/pkg/snapshot/overlay.go @@ -27,6 +27,8 @@ import ( "unsafe" "github.com/containerd/accelerated-container-image/pkg/label" + "github.com/containerd/accelerated-container-image/pkg/snapshot/diskquota" + "github.com/containerd/accelerated-container-image/pkg/metrics" "github.com/data-accelerator/zdfs" "github.com/sirupsen/logrus" @@ -91,7 +93,8 @@ type BootConfig struct { WritableLayerType string `json:"writableLayerType"` // append or sparse MirrorRegistry []Registry `json:"mirrorRegistry"` DefaultFsType string `json:"defaultFsType"` - Tenant int `json:"tenant"` // do not set this if only a single snapshotter service in the host + RootfsQuota string `json:"rootfsQuota"` // "20g" rootfs quota, only effective when rwMode is 'overlayfs' + Tenant int `json:"tenant"` // do not set this if only a single snapshotter service in the host } func DefaultBootConfig() *BootConfig { @@ -108,6 +111,7 @@ func DefaultBootConfig() *BootConfig { MirrorRegistry: nil, WritableLayerType: "append", DefaultFsType: "ext4", + RootfsQuota: "", Tenant: -1, } } @@ -177,6 +181,9 @@ type snapshotter struct { defaultFsType string tenant int locker *locker.Locker + + quotaDriver *diskquota.PrjQuotaDriver + quotaSize string } // NewSnapshotter returns a Snapshotter which uses block device based on overlayFS. @@ -229,6 +236,10 @@ func NewSnapshotter(bootConfig *BootConfig, opts ...Opt) (snapshots.Snapshotter, defaultFsType: bootConfig.DefaultFsType, locker: locker.New(), tenant: bootConfig.Tenant, + quotaSize: bootConfig.RootfsQuota, + quotaDriver: &diskquota.PrjQuotaDriver{ + QuotaIDs: make(map[uint32]struct{}), + }, }, nil } @@ -510,6 +521,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind, // If Preparing for rootfs, find metadata from its parent (top layer), launch and mount backstore device. if o.isPrepareRootfs(info) { + log.G(ctx).Infof("Preparing rootfs(%s). writeType: %s", s.ID, writeType) if writeType != RoDir { stype = storageTypeLocalBlock @@ -1142,6 +1154,15 @@ func (o *snapshotter) normalOverlayMount(s storage.Snapshot) []mount.Mount { } } +func (o *snapshotter) getDiskQuotaSize(info *snapshots.Info) string { + if info.Labels != nil { + if size, ok := info.Labels[label.RootfsQuotaLabel]; ok { + return size + } + } + return o.quotaSize +} + func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ string, _ snapshots.Info, err error) { var td, path string defer func() { @@ -1159,8 +1180,10 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k } } }() + _, tmpinfo, _, err := storage.GetInfo(ctx, key) snapshotDir := filepath.Join(o.root, "snapshots") + td, err = o.prepareDirectory(ctx, snapshotDir, kind) if err != nil { return "", snapshots.Info{}, errors.Wrap(err, "failed to create prepare snapshot dir") @@ -1182,14 +1205,29 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return "", snapshots.Info{}, errors.Wrap(err, "failed to chown") } } - + if o.isPrepareRootfs(tmpinfo) { + if diskQuotaSize := o.getDiskQuotaSize(&tmpinfo); diskQuotaSize != "" { + log.G(ctx).Infof("set usage quota %s for rootfs(sn: %s)", diskQuotaSize, s.ID) + upperPath := filepath.Join(td, "fs") + if err := o.setDiskQuota(ctx, upperPath, diskQuotaSize, diskquota.QuotaMinID); err != nil { + return "", snapshots.Info{}, errors.Wrapf(err, "failed to set diskquota on upperpath, snapshot id: %s", s.ID) + } + // if there's no parent, we just return a bind mount, so no need to set quota on workerpath + if len(s.ParentIDs) > 0 { + workpath := filepath.Join(td, "work") + if err := o.setDiskQuota(ctx, workpath, diskQuotaSize, diskquota.QuotaMinID); err != nil { + return "", snapshots.Info{}, errors.Wrapf(err, "failed to set diskquota on workpath, snapshot id: %s", s.ID) + } + } + } + } path = filepath.Join(snapshotDir, s.ID) if err = os.Rename(td, path); err != nil { return "", snapshots.Info{}, errors.Wrap(err, "failed to rename") } td = "" - id, info, _, err := storage.GetInfo(ctx, key) + if err != nil { return "", snapshots.Info{}, errors.Wrap(err, "failed to get snapshot info") } @@ -1267,6 +1305,27 @@ func (o *snapshotter) identifySnapshotStorageType(ctx context.Context, id string return st, err } +// SetDiskQuota is used to set quota for directory. +func (o *snapshotter) setDiskQuota(ctx context.Context, dir string, size string, quotaID uint32) error { + log.G(ctx).Infof("setDiskQuota: dir %s, size %s", dir, size) + if isRegular, err := diskquota.CheckRegularFile(dir); err != nil || !isRegular { + log.G(ctx).Errorf("set quota skip not regular file: %s", dir) + return err + } + + id := o.quotaDriver.GetQuotaIDInFileAttr(dir) + if id > 0 && id != quotaID { + return fmt.Errorf("quota id is already set, quota id: %d", id) + } + + log.G(ctx).Infof("try to set disk quota, dir(%s), size(%s), quotaID(%d)", dir, size, quotaID) + + if err := o.quotaDriver.SetDiskQuota(dir, size, quotaID); err != nil { + return errors.Wrapf(err, "failed to set dir(%s) disk quota", dir) + } + return nil +} + func (o *snapshotter) snPath(id string) string { return filepath.Join(o.root, "snapshots", id) } diff --git a/pkg/snapshot/storage.go b/pkg/snapshot/storage.go index a949d4df..e99c2f04 100644 --- a/pkg/snapshot/storage.go +++ b/pkg/snapshot/storage.go @@ -216,8 +216,11 @@ func (o *snapshotter) unmountAndDetachBlockDevice(ctx context.Context, snID stri // TODO(fuweid): need to track the middle state if the process has been killed. func (o *snapshotter) attachAndMountBlockDevice(ctx context.Context, snID string, writable string, fsType string, mkfs bool) (retErr error) { + log.G(ctx).Debugf("lookup device mountpoint(%s) if exists before attach.", snID) if err := lookup(o.overlaybdMountpoint(snID)); err == nil { return nil + } else { + log.G(ctx).Infof(err.Error()) } targetPath := o.overlaybdTargetPath(snID) @@ -228,6 +231,7 @@ func (o *snapshotter) attachAndMountBlockDevice(ctx context.Context, snID string defer func() { if retErr != nil { + log.G(ctx).Error(retErr.Error()) rerr := os.RemoveAll(targetPath) if rerr != nil { log.G(ctx).WithError(rerr).Warnf("failed to clean target dir %s", targetPath) @@ -236,12 +240,14 @@ func (o *snapshotter) attachAndMountBlockDevice(ctx context.Context, snID string }() if err = os.WriteFile(path.Join(targetPath, "control"), ([]byte)(fmt.Sprintf("dev_config=overlaybd/%s", o.overlaybdConfPath(snID))), 0666); err != nil { - return errors.Wrapf(err, "failed to write target dev_config for %s", targetPath) + return errors.Wrapf(err, "failed to write target dev_config for %s: %s", + targetPath, fmt.Sprintf("dev_config=overlaybd/%s", o.overlaybdConfPath(snID))) } err = os.WriteFile(path.Join(targetPath, "control"), ([]byte)(fmt.Sprintf("max_data_area_mb=%d", obdMaxDataAreaMB)), 0666) if err != nil { - return errors.Wrapf(err, "failed to write target max_data_area_mb for %s", targetPath) + return errors.Wrapf(err, "failed to write target max_data_area_mb for %s: %s", + targetPath, fmt.Sprintf("max_data_area_mb=%d", obdMaxDataAreaMB)) } err = os.WriteFile(path.Join(targetPath, "enable"), ([]byte)("1"), 0666) @@ -405,7 +411,8 @@ func (o *snapshotter) attachAndMountBlockDevice(ctx context.Context, snID string if writable != RoDir { mountFlag = 0 } - log.G(ctx).Infof("fs type: %s, mount options: %s, rw: %s", fstype, mountOpts, writable) + log.G(ctx).Infof("fs type: %s, mount options: %s, rw: %s, mountpoint: %s", + fstype, mountOpts, writable, mountPoint) if err := unix.Mount(device, mountPoint, fstype, mountFlag, mountOpts); err != nil { lastErr = errors.Wrapf(err, "failed to mount %s to %s", device, mountPoint) time.Sleep(10 * time.Millisecond) @@ -743,11 +750,11 @@ func lookup(dir string) error { m, err := mountinfo.GetMounts(mountinfo.SingleEntryFilter(dir)) if err != nil { - return errors.Wrapf(err, "failed to find the mount info for %q", dir) + return errors.Wrapf(err, "failed to get mount info for %q", dir) } if len(m) == 0 { - return errors.Errorf("failed to find the mount info for %q", dir) + return errors.Errorf("failed to find the mount point for %q", dir) } return nil }