From 88e8350de28e2bdb79a051a7b8504b21e810b341 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 7 Nov 2019 17:25:49 +0900 Subject: [PATCH 1/2] cgroup2: split fs2 from fs split fs2 package from fs, as mixing up fs and fs2 is very likely to result in unmaintainable code. Inspired by containerd/cgroups#109 Fix #2157 Signed-off-by: Akihiro Suda --- libcontainer/cgroups/fs/apply_raw.go | 64 +----- libcontainer/cgroups/fs/blkio.go | 17 +- libcontainer/cgroups/fs/blkio_test.go | 15 +- libcontainer/cgroups/fs/cpu.go | 13 +- libcontainer/cgroups/fs/cpu_test.go | 17 +- libcontainer/cgroups/fs/cpu_v2.go | 92 -------- libcontainer/cgroups/fs/cpuacct.go | 3 +- libcontainer/cgroups/fs/cpuset.go | 9 +- libcontainer/cgroups/fs/cpuset_test.go | 6 +- libcontainer/cgroups/fs/cpuset_v2.go | 162 -------------- libcontainer/cgroups/fs/devices.go | 11 +- libcontainer/cgroups/fs/devices_test.go | 7 +- libcontainer/cgroups/fs/freezer.go | 5 +- libcontainer/cgroups/fs/freezer_test.go | 3 +- libcontainer/cgroups/fs/freezer_v2.go | 74 ------- libcontainer/cgroups/fs/hugetlb.go | 9 +- libcontainer/cgroups/fs/hugetlb_test.go | 3 +- libcontainer/cgroups/fs/memory.go | 33 +-- libcontainer/cgroups/fs/memory_test.go | 23 +- libcontainer/cgroups/fs/memory_v2.go | 164 -------------- libcontainer/cgroups/fs/net_cls.go | 3 +- libcontainer/cgroups/fs/net_cls_test.go | 4 +- libcontainer/cgroups/fs/net_prio.go | 3 +- libcontainer/cgroups/fs/net_prio_test.go | 3 +- libcontainer/cgroups/fs/pids.go | 9 +- libcontainer/cgroups/fs/pids_test.go | 5 +- libcontainer/cgroups/fs/pids_v2.go | 107 --------- libcontainer/cgroups/fs/util_test.go | 3 +- libcontainer/cgroups/fs2/cpu.go | 56 +++++ libcontainer/cgroups/fs2/cpuset.go | 22 ++ libcontainer/cgroups/fs2/defaultpath.go | 99 +++++++++ libcontainer/cgroups/fs2/defaultpath_test.go | 76 +++++++ .../{fs/devices_v2.go => fs2/devices.go} | 28 +-- libcontainer/cgroups/fs2/freezer.go | 53 +++++ libcontainer/cgroups/fs2/fs2.go | 209 ++++++++++++++++++ .../cgroups/{fs/io_v2.go => fs2/io.go} | 49 +--- libcontainer/cgroups/fs2/memory.go | 103 +++++++++ libcontainer/cgroups/fs2/pids.go | 90 ++++++++ libcontainer/cgroups/fscommon/fscommon.go | 36 +++ .../cgroups/{fs => fscommon}/utils.go | 14 +- .../cgroups/{fs => fscommon}/utils_test.go | 14 +- .../cgroups/systemd/unified_hierarchy.go | 100 +++------ libcontainer/factory_linux.go | 32 +++ 43 files changed, 951 insertions(+), 897 deletions(-) delete mode 100644 libcontainer/cgroups/fs/cpu_v2.go delete mode 100644 libcontainer/cgroups/fs/cpuset_v2.go delete mode 100644 libcontainer/cgroups/fs/freezer_v2.go delete mode 100644 libcontainer/cgroups/fs/memory_v2.go delete mode 100644 libcontainer/cgroups/fs/pids_v2.go create mode 100644 libcontainer/cgroups/fs2/cpu.go create mode 100644 libcontainer/cgroups/fs2/cpuset.go create mode 100644 libcontainer/cgroups/fs2/defaultpath.go create mode 100644 libcontainer/cgroups/fs2/defaultpath_test.go rename libcontainer/cgroups/{fs/devices_v2.go => fs2/devices.go} (70%) create mode 100644 libcontainer/cgroups/fs2/freezer.go create mode 100644 libcontainer/cgroups/fs2/fs2.go rename libcontainer/cgroups/{fs/io_v2.go => fs2/io.go} (62%) create mode 100644 libcontainer/cgroups/fs2/memory.go create mode 100644 libcontainer/cgroups/fs2/pids.go create mode 100644 libcontainer/cgroups/fscommon/fscommon.go rename libcontainer/cgroups/{fs => fscommon}/utils.go (83%) rename libcontainer/cgroups/{fs => fscommon}/utils_test.go (84%) diff --git a/libcontainer/cgroups/fs/apply_raw.go b/libcontainer/cgroups/fs/apply_raw.go index fc40bf55560..47f575f68fc 100644 --- a/libcontainer/cgroups/fs/apply_raw.go +++ b/libcontainer/cgroups/fs/apply_raw.go @@ -5,7 +5,6 @@ package fs import ( "fmt" "io" - "io/ioutil" "os" "path/filepath" "sync" @@ -33,15 +32,6 @@ var ( &FreezerGroup{}, &NameGroup{GroupName: "name=systemd", Join: true}, } - subsystemsUnified = subsystemSet{ - &CpusetGroupV2{}, - &FreezerGroupV2{}, - &CpuGroupV2{}, - &MemoryGroupV2{}, - &IOGroupV2{}, - &PidsGroupV2{}, - &DevicesGroupV2{}, - } HugePageSizes, _ = cgroups.GetHugePageSize() ) @@ -139,9 +129,6 @@ func isIgnorableError(rootless bool, err error) bool { } func (m *Manager) getSubsystems() subsystemSet { - if cgroups.IsCgroup2UnifiedMode() { - return subsystemsUnified - } return subsystemsLegacy } @@ -226,25 +213,7 @@ func (m *Manager) GetPaths() map[string]string { } func (m *Manager) GetUnifiedPath() (string, error) { - if !cgroups.IsCgroup2UnifiedMode() { - return "", errors.New("unified path is only supported when running in unified mode") - } - unifiedPath := "" - m.mu.Lock() - defer m.mu.Unlock() - for k, v := range m.Paths { - if unifiedPath == "" { - unifiedPath = v - } else if v != unifiedPath { - return unifiedPath, - errors.Errorf("expected %q path to be unified path %q, got %q", k, unifiedPath, v) - } - } - if unifiedPath == "" { - // FIXME: unified path could be detected even when no controller is available - return unifiedPath, errors.New("cannot detect unified path") - } - return unifiedPath, nil + return "", errors.New("unified path is only supported when running in unified mode") } func (m *Manager) GetStats() (*cgroups.Stats, error) { @@ -325,25 +294,11 @@ func (m *Manager) Freeze(state configs.FreezerState) error { } func (m *Manager) GetPids() ([]int, error) { - if cgroups.IsCgroup2UnifiedMode() { - path, err := m.GetUnifiedPath() - if err != nil { - return nil, err - } - return cgroups.GetPids(path) - } paths := m.GetPaths() return cgroups.GetPids(paths["devices"]) } func (m *Manager) GetAllPids() ([]int, error) { - if cgroups.IsCgroup2UnifiedMode() { - path, err := m.GetUnifiedPath() - if err != nil { - return nil, err - } - return cgroups.GetAllPids(path) - } paths := m.GetPaths() return cgroups.GetAllPids(paths["devices"]) } @@ -414,23 +369,6 @@ func (raw *cgroupData) join(subsystem string) (string, error) { return path, nil } -func writeFile(dir, file, data string) error { - // Normally dir should not be empty, one case is that cgroup subsystem - // is not mounted, we will get empty dir, and we want it fail here. - if dir == "" { - return fmt.Errorf("no such directory for %s", file) - } - if err := ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700); err != nil { - return fmt.Errorf("failed to write %v to %v: %v", data, file, err) - } - return nil -} - -func readFile(dir, file string) (string, error) { - data, err := ioutil.ReadFile(filepath.Join(dir, file)) - return string(data), err -} - func removePath(p string, err error) error { if err != nil { return err diff --git a/libcontainer/cgroups/fs/blkio.go b/libcontainer/cgroups/fs/blkio.go index a142cb991df..52c118d68c1 100644 --- a/libcontainer/cgroups/fs/blkio.go +++ b/libcontainer/cgroups/fs/blkio.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -31,41 +32,41 @@ func (s *BlkioGroup) Apply(d *cgroupData) error { func (s *BlkioGroup) Set(path string, cgroup *configs.Cgroup) error { if cgroup.Resources.BlkioWeight != 0 { - if err := writeFile(path, "blkio.weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioWeight), 10)); err != nil { + if err := fscommon.WriteFile(path, "blkio.weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioWeight), 10)); err != nil { return err } } if cgroup.Resources.BlkioLeafWeight != 0 { - if err := writeFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioLeafWeight), 10)); err != nil { + if err := fscommon.WriteFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioLeafWeight), 10)); err != nil { return err } } for _, wd := range cgroup.Resources.BlkioWeightDevice { - if err := writeFile(path, "blkio.weight_device", wd.WeightString()); err != nil { + if err := fscommon.WriteFile(path, "blkio.weight_device", wd.WeightString()); err != nil { return err } - if err := writeFile(path, "blkio.leaf_weight_device", wd.LeafWeightString()); err != nil { + if err := fscommon.WriteFile(path, "blkio.leaf_weight_device", wd.LeafWeightString()); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleReadBpsDevice { - if err := writeFile(path, "blkio.throttle.read_bps_device", td.String()); err != nil { + if err := fscommon.WriteFile(path, "blkio.throttle.read_bps_device", td.String()); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleWriteBpsDevice { - if err := writeFile(path, "blkio.throttle.write_bps_device", td.String()); err != nil { + if err := fscommon.WriteFile(path, "blkio.throttle.write_bps_device", td.String()); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleReadIOPSDevice { - if err := writeFile(path, "blkio.throttle.read_iops_device", td.String()); err != nil { + if err := fscommon.WriteFile(path, "blkio.throttle.read_iops_device", td.String()); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleWriteIOPSDevice { - if err := writeFile(path, "blkio.throttle.write_iops_device", td.String()); err != nil { + if err := fscommon.WriteFile(path, "blkio.throttle.write_iops_device", td.String()); err != nil { return err } } diff --git a/libcontainer/cgroups/fs/blkio_test.go b/libcontainer/cgroups/fs/blkio_test.go index 6957392048d..5ba60fa7f7b 100644 --- a/libcontainer/cgroups/fs/blkio_test.go +++ b/libcontainer/cgroups/fs/blkio_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -95,7 +96,7 @@ func TestBlkioSetWeight(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "blkio.weight") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "blkio.weight") if err != nil { t.Fatalf("Failed to parse blkio.weight - %s", err) } @@ -126,7 +127,7 @@ func TestBlkioSetWeightDevice(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "blkio.weight_device") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "blkio.weight_device") if err != nil { t.Fatalf("Failed to parse blkio.weight_device - %s", err) } @@ -163,7 +164,7 @@ func TestBlkioSetMultipleWeightDevice(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "blkio.weight_device") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "blkio.weight_device") if err != nil { t.Fatalf("Failed to parse blkio.weight_device - %s", err) } @@ -535,7 +536,7 @@ func TestBlkioSetThrottleReadBpsDevice(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.read_bps_device") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "blkio.throttle.read_bps_device") if err != nil { t.Fatalf("Failed to parse blkio.throttle.read_bps_device - %s", err) } @@ -565,7 +566,7 @@ func TestBlkioSetThrottleWriteBpsDevice(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.write_bps_device") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "blkio.throttle.write_bps_device") if err != nil { t.Fatalf("Failed to parse blkio.throttle.write_bps_device - %s", err) } @@ -595,7 +596,7 @@ func TestBlkioSetThrottleReadIOpsDevice(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.read_iops_device") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "blkio.throttle.read_iops_device") if err != nil { t.Fatalf("Failed to parse blkio.throttle.read_iops_device - %s", err) } @@ -625,7 +626,7 @@ func TestBlkioSetThrottleWriteIOpsDevice(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.write_iops_device") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "blkio.throttle.write_iops_device") if err != nil { t.Fatalf("Failed to parse blkio.throttle.write_iops_device - %s", err) } diff --git a/libcontainer/cgroups/fs/cpu.go b/libcontainer/cgroups/fs/cpu.go index e240a8313a2..4db7b647cf0 100644 --- a/libcontainer/cgroups/fs/cpu.go +++ b/libcontainer/cgroups/fs/cpu.go @@ -9,6 +9,7 @@ import ( "strconv" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -51,12 +52,12 @@ func (s *CpuGroup) ApplyDir(path string, cgroup *configs.Cgroup, pid int) error func (s *CpuGroup) SetRtSched(path string, cgroup *configs.Cgroup) error { if cgroup.Resources.CpuRtPeriod != 0 { - if err := writeFile(path, "cpu.rt_period_us", strconv.FormatUint(cgroup.Resources.CpuRtPeriod, 10)); err != nil { + if err := fscommon.WriteFile(path, "cpu.rt_period_us", strconv.FormatUint(cgroup.Resources.CpuRtPeriod, 10)); err != nil { return err } } if cgroup.Resources.CpuRtRuntime != 0 { - if err := writeFile(path, "cpu.rt_runtime_us", strconv.FormatInt(cgroup.Resources.CpuRtRuntime, 10)); err != nil { + if err := fscommon.WriteFile(path, "cpu.rt_runtime_us", strconv.FormatInt(cgroup.Resources.CpuRtRuntime, 10)); err != nil { return err } } @@ -65,17 +66,17 @@ func (s *CpuGroup) SetRtSched(path string, cgroup *configs.Cgroup) error { func (s *CpuGroup) Set(path string, cgroup *configs.Cgroup) error { if cgroup.Resources.CpuShares != 0 { - if err := writeFile(path, "cpu.shares", strconv.FormatUint(cgroup.Resources.CpuShares, 10)); err != nil { + if err := fscommon.WriteFile(path, "cpu.shares", strconv.FormatUint(cgroup.Resources.CpuShares, 10)); err != nil { return err } } if cgroup.Resources.CpuPeriod != 0 { - if err := writeFile(path, "cpu.cfs_period_us", strconv.FormatUint(cgroup.Resources.CpuPeriod, 10)); err != nil { + if err := fscommon.WriteFile(path, "cpu.cfs_period_us", strconv.FormatUint(cgroup.Resources.CpuPeriod, 10)); err != nil { return err } } if cgroup.Resources.CpuQuota != 0 { - if err := writeFile(path, "cpu.cfs_quota_us", strconv.FormatInt(cgroup.Resources.CpuQuota, 10)); err != nil { + if err := fscommon.WriteFile(path, "cpu.cfs_quota_us", strconv.FormatInt(cgroup.Resources.CpuQuota, 10)); err != nil { return err } } @@ -98,7 +99,7 @@ func (s *CpuGroup) GetStats(path string, stats *cgroups.Stats) error { sc := bufio.NewScanner(f) for sc.Scan() { - t, v, err := getCgroupParamKeyValue(sc.Text()) + t, v, err := fscommon.GetCgroupParamKeyValue(sc.Text()) if err != nil { return err } diff --git a/libcontainer/cgroups/fs/cpu_test.go b/libcontainer/cgroups/fs/cpu_test.go index 6369c91ad65..2eeb489efba 100644 --- a/libcontainer/cgroups/fs/cpu_test.go +++ b/libcontainer/cgroups/fs/cpu_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" ) func TestCpuSetShares(t *testing.T) { @@ -29,7 +30,7 @@ func TestCpuSetShares(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "cpu.shares") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.shares") if err != nil { t.Fatalf("Failed to parse cpu.shares - %s", err) } @@ -70,7 +71,7 @@ func TestCpuSetBandWidth(t *testing.T) { t.Fatal(err) } - quota, err := getCgroupParamUint(helper.CgroupPath, "cpu.cfs_quota_us") + quota, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.cfs_quota_us") if err != nil { t.Fatalf("Failed to parse cpu.cfs_quota_us - %s", err) } @@ -78,21 +79,21 @@ func TestCpuSetBandWidth(t *testing.T) { t.Fatal("Got the wrong value, set cpu.cfs_quota_us failed.") } - period, err := getCgroupParamUint(helper.CgroupPath, "cpu.cfs_period_us") + period, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.cfs_period_us") if err != nil { t.Fatalf("Failed to parse cpu.cfs_period_us - %s", err) } if period != periodAfter { t.Fatal("Got the wrong value, set cpu.cfs_period_us failed.") } - rtRuntime, err := getCgroupParamUint(helper.CgroupPath, "cpu.rt_runtime_us") + rtRuntime, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.rt_runtime_us") if err != nil { t.Fatalf("Failed to parse cpu.rt_runtime_us - %s", err) } if rtRuntime != rtRuntimeAfter { t.Fatal("Got the wrong value, set cpu.rt_runtime_us failed.") } - rtPeriod, err := getCgroupParamUint(helper.CgroupPath, "cpu.rt_period_us") + rtPeriod, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.rt_period_us") if err != nil { t.Fatalf("Failed to parse cpu.rt_period_us - %s", err) } @@ -185,21 +186,21 @@ func TestCpuSetRtSchedAtApply(t *testing.T) { t.Fatal(err) } - rtRuntime, err := getCgroupParamUint(helper.CgroupPath, "cpu.rt_runtime_us") + rtRuntime, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.rt_runtime_us") if err != nil { t.Fatalf("Failed to parse cpu.rt_runtime_us - %s", err) } if rtRuntime != rtRuntimeAfter { t.Fatal("Got the wrong value, set cpu.rt_runtime_us failed.") } - rtPeriod, err := getCgroupParamUint(helper.CgroupPath, "cpu.rt_period_us") + rtPeriod, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cpu.rt_period_us") if err != nil { t.Fatalf("Failed to parse cpu.rt_period_us - %s", err) } if rtPeriod != rtPeriodAfter { t.Fatal("Got the wrong value, set cpu.rt_period_us failed.") } - pid, err := getCgroupParamUint(helper.CgroupPath, "cgroup.procs") + pid, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "cgroup.procs") if err != nil { t.Fatalf("Failed to parse cgroup.procs - %s", err) } diff --git a/libcontainer/cgroups/fs/cpu_v2.go b/libcontainer/cgroups/fs/cpu_v2.go deleted file mode 100644 index 245071ae504..00000000000 --- a/libcontainer/cgroups/fs/cpu_v2.go +++ /dev/null @@ -1,92 +0,0 @@ -// +build linux - -package fs - -import ( - "bufio" - "os" - "path/filepath" - "strconv" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/configs" -) - -type CpuGroupV2 struct { -} - -func (s *CpuGroupV2) Name() string { - return "cpu" -} - -func (s *CpuGroupV2) Apply(d *cgroupData) error { - // We always want to join the cpu group, to allow fair cpu scheduling - // on a container basis - path, err := d.path("cpu") - if err != nil && !cgroups.IsNotFound(err) { - return err - } - return s.ApplyDir(path, d.config, d.pid) -} - -func (s *CpuGroupV2) ApplyDir(path string, cgroup *configs.Cgroup, pid int) error { - // This might happen if we have no cpu cgroup mounted. - // Just do nothing and don't fail. - if path == "" { - return nil - } - if err := os.MkdirAll(path, 0755); err != nil { - return err - } - return cgroups.WriteCgroupProc(path, pid) -} - -func (s *CpuGroupV2) Set(path string, cgroup *configs.Cgroup) error { - if cgroup.Resources.CpuWeight != 0 { - if err := writeFile(path, "cpu.weight", strconv.FormatUint(cgroup.Resources.CpuWeight, 10)); err != nil { - return err - } - } - - if cgroup.Resources.CpuMax != "" { - if err := writeFile(path, "cpu.max", cgroup.Resources.CpuMax); err != nil { - return err - } - } - - return nil -} - -func (s *CpuGroupV2) Remove(d *cgroupData) error { - return removePath(d.path("cpu")) -} - -func (s *CpuGroupV2) GetStats(path string, stats *cgroups.Stats) error { - f, err := os.Open(filepath.Join(path, "cpu.stat")) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer f.Close() - - sc := bufio.NewScanner(f) - for sc.Scan() { - t, v, err := getCgroupParamKeyValue(sc.Text()) - if err != nil { - return err - } - switch t { - case "usage_usec": - stats.CpuStats.CpuUsage.TotalUsage = v * 1000 - - case "user_usec": - stats.CpuStats.CpuUsage.UsageInUsermode = v * 1000 - - case "system_usec": - stats.CpuStats.CpuUsage.UsageInKernelmode = v * 1000 - } - } - return nil -} diff --git a/libcontainer/cgroups/fs/cpuacct.go b/libcontainer/cgroups/fs/cpuacct.go index 032b76ecf08..95dc9a1ecc5 100644 --- a/libcontainer/cgroups/fs/cpuacct.go +++ b/libcontainer/cgroups/fs/cpuacct.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/system" ) @@ -51,7 +52,7 @@ func (s *CpuacctGroup) GetStats(path string, stats *cgroups.Stats) error { return err } - totalUsage, err := getCgroupParamUint(path, "cpuacct.usage") + totalUsage, err := fscommon.GetCgroupParamUint(path, "cpuacct.usage") if err != nil { return err } diff --git a/libcontainer/cgroups/fs/cpuset.go b/libcontainer/cgroups/fs/cpuset.go index 5a1d152ea1d..bfc900e3f6c 100644 --- a/libcontainer/cgroups/fs/cpuset.go +++ b/libcontainer/cgroups/fs/cpuset.go @@ -10,6 +10,7 @@ import ( "path/filepath" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" ) @@ -31,12 +32,12 @@ func (s *CpusetGroup) Apply(d *cgroupData) error { func (s *CpusetGroup) Set(path string, cgroup *configs.Cgroup) error { if cgroup.Resources.CpusetCpus != "" { - if err := writeFile(path, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil { + if err := fscommon.WriteFile(path, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil { return err } } if cgroup.Resources.CpusetMems != "" { - if err := writeFile(path, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil { + if err := fscommon.WriteFile(path, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil { return err } } @@ -135,12 +136,12 @@ func (s *CpusetGroup) copyIfNeeded(current, parent string) error { } if s.isEmpty(currentCpus) { - if err := writeFile(current, "cpuset.cpus", string(parentCpus)); err != nil { + if err := fscommon.WriteFile(current, "cpuset.cpus", string(parentCpus)); err != nil { return err } } if s.isEmpty(currentMems) { - if err := writeFile(current, "cpuset.mems", string(parentMems)); err != nil { + if err := fscommon.WriteFile(current, "cpuset.mems", string(parentMems)); err != nil { return err } } diff --git a/libcontainer/cgroups/fs/cpuset_test.go b/libcontainer/cgroups/fs/cpuset_test.go index 0f929151fc7..927e6311082 100644 --- a/libcontainer/cgroups/fs/cpuset_test.go +++ b/libcontainer/cgroups/fs/cpuset_test.go @@ -4,6 +4,8 @@ package fs import ( "testing" + + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" ) func TestCpusetSetCpus(t *testing.T) { @@ -25,7 +27,7 @@ func TestCpusetSetCpus(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "cpuset.cpus") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "cpuset.cpus") if err != nil { t.Fatalf("Failed to parse cpuset.cpus - %s", err) } @@ -54,7 +56,7 @@ func TestCpusetSetMems(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "cpuset.mems") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "cpuset.mems") if err != nil { t.Fatalf("Failed to parse cpuset.mems - %s", err) } diff --git a/libcontainer/cgroups/fs/cpuset_v2.go b/libcontainer/cgroups/fs/cpuset_v2.go deleted file mode 100644 index 35c194be1f8..00000000000 --- a/libcontainer/cgroups/fs/cpuset_v2.go +++ /dev/null @@ -1,162 +0,0 @@ -// +build linux - -package fs - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/configs" - libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" -) - -type CpusetGroupV2 struct { -} - -func (s *CpusetGroupV2) Name() string { - return "cpuset" -} - -func (s *CpusetGroupV2) Apply(d *cgroupData) error { - if d.config.Resources.CpusetCpus == "" && d.config.Resources.CpusetMems == "" { - return nil - } - dir, err := d.path("cpuset") - if err != nil && !cgroups.IsNotFound(err) { - return err - } - return s.applyDir(dir, d.config, d.pid) -} - -func (s *CpusetGroupV2) Set(path string, cgroup *configs.Cgroup) error { - if cgroup.Resources.CpusetCpus != "" { - if err := writeFile(path, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil { - return err - } - } - if cgroup.Resources.CpusetMems != "" { - if err := writeFile(path, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil { - return err - } - } - return nil -} - -func (s *CpusetGroupV2) Remove(d *cgroupData) error { - return removePath(d.path("cpuset")) -} - -func (s *CpusetGroupV2) GetStats(path string, stats *cgroups.Stats) error { - return nil -} - -func (s *CpusetGroupV2) applyDir(dir string, cgroup *configs.Cgroup, pid int) error { - // This might happen if we have no cpuset cgroup mounted. - // Just do nothing and don't fail. - if dir == "" { - return nil - } - mountInfo, err := ioutil.ReadFile("/proc/self/mountinfo") - if err != nil { - return err - } - root := filepath.Dir(cgroups.GetClosestMountpointAncestor(dir, string(mountInfo))) - // 'ensureParent' start with parent because we don't want to - // explicitly inherit from parent, it could conflict with - // 'cpuset.cpu_exclusive'. - if err := s.ensureParent(filepath.Dir(dir), root); err != nil { - return err - } - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - // We didn't inherit cpuset configs from parent, but we have - // to ensure cpuset configs are set before moving task into the - // cgroup. - // The logic is, if user specified cpuset configs, use these - // specified configs, otherwise, inherit from parent. This makes - // cpuset configs work correctly with 'cpuset.cpu_exclusive', and - // keep backward compatibility. - if err := s.ensureCpusAndMems(dir, cgroup); err != nil { - return err - } - - // because we are not using d.join we need to place the pid into the procs file - // unlike the other subsystems - return cgroups.WriteCgroupProc(dir, pid) -} - -func (s *CpusetGroupV2) getSubsystemSettings(parent string) (cpus []byte, mems []byte, err error) { - if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus.effective")); err != nil { - return - } - if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems.effective")); err != nil { - return - } - return cpus, mems, nil -} - -// ensureParent makes sure that the parent directory of current is created -// and populated with the proper cpus and mems files copied from -// it's parent. -func (s *CpusetGroupV2) ensureParent(current, root string) error { - parent := filepath.Dir(current) - if libcontainerUtils.CleanPath(parent) == root { - return nil - } - // Avoid infinite recursion. - if parent == current { - return fmt.Errorf("cpuset: cgroup parent path outside cgroup root") - } - if err := s.ensureParent(parent, root); err != nil { - return err - } - if err := os.MkdirAll(current, 0755); err != nil { - return err - } - return s.copyIfNeeded(current, parent) -} - -// copyIfNeeded copies the cpuset.cpus and cpuset.mems from the parent -// directory to the current directory if the file's contents are 0 -func (s *CpusetGroupV2) copyIfNeeded(current, parent string) error { - var ( - err error - currentCpus, currentMems []byte - parentCpus, parentMems []byte - ) - - if currentCpus, currentMems, err = s.getSubsystemSettings(current); err != nil { - return err - } - if parentCpus, parentMems, err = s.getSubsystemSettings(parent); err != nil { - return err - } - - if s.isEmpty(currentCpus) { - if err := writeFile(current, "cpuset.cpus", string(parentCpus)); err != nil { - return err - } - } - if s.isEmpty(currentMems) { - if err := writeFile(current, "cpuset.mems", string(parentMems)); err != nil { - return err - } - } - return nil -} - -func (s *CpusetGroupV2) isEmpty(b []byte) bool { - return len(bytes.Trim(b, "\n")) == 0 -} - -func (s *CpusetGroupV2) ensureCpusAndMems(path string, cgroup *configs.Cgroup) error { - if err := s.Set(path, cgroup); err != nil { - return err - } - return s.copyIfNeeded(path, filepath.Dir(path)) -} diff --git a/libcontainer/cgroups/fs/devices.go b/libcontainer/cgroups/fs/devices.go index 0ac5b4ed700..036c8db773e 100644 --- a/libcontainer/cgroups/fs/devices.go +++ b/libcontainer/cgroups/fs/devices.go @@ -4,6 +4,7 @@ package fs import ( "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/system" ) @@ -37,7 +38,7 @@ func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error { if dev.Allow { file = "devices.allow" } - if err := writeFile(path, file, dev.CgroupString()); err != nil { + if err := fscommon.WriteFile(path, file, dev.CgroupString()); err != nil { return err } } @@ -45,25 +46,25 @@ func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error { } if cgroup.Resources.AllowAllDevices != nil { if *cgroup.Resources.AllowAllDevices == false { - if err := writeFile(path, "devices.deny", "a"); err != nil { + if err := fscommon.WriteFile(path, "devices.deny", "a"); err != nil { return err } for _, dev := range cgroup.Resources.AllowedDevices { - if err := writeFile(path, "devices.allow", dev.CgroupString()); err != nil { + if err := fscommon.WriteFile(path, "devices.allow", dev.CgroupString()); err != nil { return err } } return nil } - if err := writeFile(path, "devices.allow", "a"); err != nil { + if err := fscommon.WriteFile(path, "devices.allow", "a"); err != nil { return err } } for _, dev := range cgroup.Resources.DeniedDevices { - if err := writeFile(path, "devices.deny", dev.CgroupString()); err != nil { + if err := fscommon.WriteFile(path, "devices.deny", dev.CgroupString()); err != nil { return err } } diff --git a/libcontainer/cgroups/fs/devices_test.go b/libcontainer/cgroups/fs/devices_test.go index fc635b990f5..648f4a2ff47 100644 --- a/libcontainer/cgroups/fs/devices_test.go +++ b/libcontainer/cgroups/fs/devices_test.go @@ -5,6 +5,7 @@ package fs import ( "testing" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -48,7 +49,7 @@ func TestDevicesSetAllow(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "devices.allow") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "devices.allow") if err != nil { t.Fatalf("Failed to parse devices.allow - %s", err) } @@ -62,7 +63,7 @@ func TestDevicesSetAllow(t *testing.T) { if err := devices.Set(helper.CgroupPath, helper.CgroupData.config); err != nil { t.Fatal(err) } - value, err = getCgroupParamString(helper.CgroupPath, "devices.allow") + value, err = fscommon.GetCgroupParamString(helper.CgroupPath, "devices.allow") if err != nil { t.Fatalf("Failed to parse devices.allow - %s", err) } @@ -87,7 +88,7 @@ func TestDevicesSetDeny(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "devices.deny") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "devices.deny") if err != nil { t.Fatalf("Failed to parse devices.deny - %s", err) } diff --git a/libcontainer/cgroups/fs/freezer.go b/libcontainer/cgroups/fs/freezer.go index 4b19f8a970d..9dc81bdb84c 100644 --- a/libcontainer/cgroups/fs/freezer.go +++ b/libcontainer/cgroups/fs/freezer.go @@ -8,6 +8,7 @@ import ( "time" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -34,11 +35,11 @@ func (s *FreezerGroup) Set(path string, cgroup *configs.Cgroup) error { // state, let's write again this state, hoping it's going to be properly // set this time. Otherwise, this loop could run infinitely, waiting for // a state change that would never happen. - if err := writeFile(path, "freezer.state", string(cgroup.Resources.Freezer)); err != nil { + if err := fscommon.WriteFile(path, "freezer.state", string(cgroup.Resources.Freezer)); err != nil { return err } - state, err := readFile(path, "freezer.state") + state, err := fscommon.ReadFile(path, "freezer.state") if err != nil { return err } diff --git a/libcontainer/cgroups/fs/freezer_test.go b/libcontainer/cgroups/fs/freezer_test.go index 77708db9a62..ad80261c805 100644 --- a/libcontainer/cgroups/fs/freezer_test.go +++ b/libcontainer/cgroups/fs/freezer_test.go @@ -5,6 +5,7 @@ package fs import ( "testing" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -22,7 +23,7 @@ func TestFreezerSetState(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "freezer.state") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "freezer.state") if err != nil { t.Fatalf("Failed to parse freezer.state - %s", err) } diff --git a/libcontainer/cgroups/fs/freezer_v2.go b/libcontainer/cgroups/fs/freezer_v2.go deleted file mode 100644 index 186de9ab41f..00000000000 --- a/libcontainer/cgroups/fs/freezer_v2.go +++ /dev/null @@ -1,74 +0,0 @@ -// +build linux - -package fs - -import ( - "fmt" - "strings" - "time" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/configs" -) - -type FreezerGroupV2 struct { -} - -func (s *FreezerGroupV2) Name() string { - return "freezer" -} - -func (s *FreezerGroupV2) Apply(d *cgroupData) error { - _, err := d.join("freezer") - if err != nil && !cgroups.IsNotFound(err) { - return err - } - return nil -} - -func (s *FreezerGroupV2) Set(path string, cgroup *configs.Cgroup) error { - var desiredState string - filename := "cgroup.freeze" - if cgroup.Resources.Freezer == configs.Frozen { - desiredState = "1" - } else { - desiredState = "0" - } - - switch cgroup.Resources.Freezer { - case configs.Frozen, configs.Thawed: - for { - // In case this loop does not exit because it doesn't get the expected - // state, let's write again this state, hoping it's going to be properly - // set this time. Otherwise, this loop could run infinitely, waiting for - // a state change that would never happen. - if err := writeFile(path, filename, desiredState); err != nil { - return err - } - - state, err := readFile(path, filename) - if err != nil { - return err - } - if strings.TrimSpace(state) == desiredState { - break - } - - time.Sleep(1 * time.Millisecond) - } - case configs.Undefined: - return nil - default: - return fmt.Errorf("Invalid argument '%s' to freezer.state", string(cgroup.Resources.Freezer)) - } - - return nil -} - -func (s *FreezerGroupV2) Remove(d *cgroupData) error { - return removePath(d.path("freezer")) -} - -func (s *FreezerGroupV2) GetStats(path string, stats *cgroups.Stats) error { - return nil -} diff --git a/libcontainer/cgroups/fs/hugetlb.go b/libcontainer/cgroups/fs/hugetlb.go index 2f9727719d0..68719c2e6de 100644 --- a/libcontainer/cgroups/fs/hugetlb.go +++ b/libcontainer/cgroups/fs/hugetlb.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -28,7 +29,7 @@ func (s *HugetlbGroup) Apply(d *cgroupData) error { func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error { for _, hugetlb := range cgroup.Resources.HugetlbLimit { - if err := writeFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.FormatUint(hugetlb.Limit, 10)); err != nil { + if err := fscommon.WriteFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.FormatUint(hugetlb.Limit, 10)); err != nil { return err } } @@ -44,21 +45,21 @@ func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error { hugetlbStats := cgroups.HugetlbStats{} for _, pageSize := range HugePageSizes { usage := strings.Join([]string{"hugetlb", pageSize, "usage_in_bytes"}, ".") - value, err := getCgroupParamUint(path, usage) + value, err := fscommon.GetCgroupParamUint(path, usage) if err != nil { return fmt.Errorf("failed to parse %s - %v", usage, err) } hugetlbStats.Usage = value maxUsage := strings.Join([]string{"hugetlb", pageSize, "max_usage_in_bytes"}, ".") - value, err = getCgroupParamUint(path, maxUsage) + value, err = fscommon.GetCgroupParamUint(path, maxUsage) if err != nil { return fmt.Errorf("failed to parse %s - %v", maxUsage, err) } hugetlbStats.MaxUsage = value failcnt := strings.Join([]string{"hugetlb", pageSize, "failcnt"}, ".") - value, err = getCgroupParamUint(path, failcnt) + value, err = fscommon.GetCgroupParamUint(path, failcnt) if err != nil { return fmt.Errorf("failed to parse %s - %v", failcnt, err) } diff --git a/libcontainer/cgroups/fs/hugetlb_test.go b/libcontainer/cgroups/fs/hugetlb_test.go index 2d41c4eb205..9ddacfee687 100644 --- a/libcontainer/cgroups/fs/hugetlb_test.go +++ b/libcontainer/cgroups/fs/hugetlb_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -54,7 +55,7 @@ func TestHugetlbSetHugetlb(t *testing.T) { for _, pageSize := range HugePageSizes { limit := fmt.Sprintf(limit, pageSize) - value, err := getCgroupParamUint(helper.CgroupPath, limit) + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, limit) if err != nil { t.Fatalf("Failed to parse %s - %s", limit, err) } diff --git a/libcontainer/cgroups/fs/memory.go b/libcontainer/cgroups/fs/memory.go index d5310d569f2..f81ed050b2a 100644 --- a/libcontainer/cgroups/fs/memory.go +++ b/libcontainer/cgroups/fs/memory.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -84,28 +85,28 @@ func setMemoryAndSwap(path string, cgroup *configs.Cgroup) error { // for memory and swap memory, so it won't fail because the new // value and the old value don't fit kernel's validation. if cgroup.Resources.MemorySwap == -1 || memoryUsage.Limit < uint64(cgroup.Resources.MemorySwap) { - if err := writeFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { + if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { return err } - if err := writeFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { + if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { return err } } else { - if err := writeFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { + if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { return err } - if err := writeFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { + if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { return err } } } else { if cgroup.Resources.Memory != 0 { - if err := writeFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { + if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { return err } } if cgroup.Resources.MemorySwap != 0 { - if err := writeFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { + if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { return err } } @@ -126,25 +127,25 @@ func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error { } if cgroup.Resources.MemoryReservation != 0 { - if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil { + if err := fscommon.WriteFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil { return err } } if cgroup.Resources.KernelMemoryTCP != 0 { - if err := writeFile(path, "memory.kmem.tcp.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemoryTCP, 10)); err != nil { + if err := fscommon.WriteFile(path, "memory.kmem.tcp.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemoryTCP, 10)); err != nil { return err } } if cgroup.Resources.OomKillDisable { - if err := writeFile(path, "memory.oom_control", "1"); err != nil { + if err := fscommon.WriteFile(path, "memory.oom_control", "1"); err != nil { return err } } if cgroup.Resources.MemorySwappiness == nil || int64(*cgroup.Resources.MemorySwappiness) == -1 { return nil } else if *cgroup.Resources.MemorySwappiness <= 100 { - if err := writeFile(path, "memory.swappiness", strconv.FormatUint(*cgroup.Resources.MemorySwappiness, 10)); err != nil { + if err := fscommon.WriteFile(path, "memory.swappiness", strconv.FormatUint(*cgroup.Resources.MemorySwappiness, 10)); err != nil { return err } } else { @@ -171,7 +172,7 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error { sc := bufio.NewScanner(statsFile) for sc.Scan() { - t, v, err := getCgroupParamKeyValue(sc.Text()) + t, v, err := fscommon.GetCgroupParamKeyValue(sc.Text()) if err != nil { return fmt.Errorf("failed to parse memory.stat (%q) - %v", sc.Text(), err) } @@ -201,7 +202,7 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error { stats.MemoryStats.KernelTCPUsage = kernelTCPUsage useHierarchy := strings.Join([]string{"memory", "use_hierarchy"}, ".") - value, err := getCgroupParamUint(path, useHierarchy) + value, err := fscommon.GetCgroupParamUint(path, useHierarchy) if err != nil { return err } @@ -233,7 +234,7 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) { failcnt := strings.Join([]string{moduleName, "failcnt"}, ".") limit := strings.Join([]string{moduleName, "limit_in_bytes"}, ".") - value, err := getCgroupParamUint(path, usage) + value, err := fscommon.GetCgroupParamUint(path, usage) if err != nil { if moduleName != "memory" && os.IsNotExist(err) { return cgroups.MemoryData{}, nil @@ -241,7 +242,7 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) { return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", usage, err) } memoryData.Usage = value - value, err = getCgroupParamUint(path, maxUsage) + value, err = fscommon.GetCgroupParamUint(path, maxUsage) if err != nil { if moduleName != "memory" && os.IsNotExist(err) { return cgroups.MemoryData{}, nil @@ -249,7 +250,7 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) { return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", maxUsage, err) } memoryData.MaxUsage = value - value, err = getCgroupParamUint(path, failcnt) + value, err = fscommon.GetCgroupParamUint(path, failcnt) if err != nil { if moduleName != "memory" && os.IsNotExist(err) { return cgroups.MemoryData{}, nil @@ -257,7 +258,7 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) { return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", failcnt, err) } memoryData.Failcnt = value - value, err = getCgroupParamUint(path, limit) + value, err = fscommon.GetCgroupParamUint(path, limit) if err != nil { if moduleName != "memory" && os.IsNotExist(err) { return cgroups.MemoryData{}, nil diff --git a/libcontainer/cgroups/fs/memory_test.go b/libcontainer/cgroups/fs/memory_test.go index 4e0ae7b26e5..62de5637701 100644 --- a/libcontainer/cgroups/fs/memory_test.go +++ b/libcontainer/cgroups/fs/memory_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" ) const ( @@ -42,7 +43,7 @@ func TestMemorySetMemory(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.limit_in_bytes - %s", err) } @@ -50,7 +51,7 @@ func TestMemorySetMemory(t *testing.T) { t.Fatal("Got the wrong value, set memory.limit_in_bytes failed.") } - value, err = getCgroupParamUint(helper.CgroupPath, "memory.soft_limit_in_bytes") + value, err = fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.soft_limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.soft_limit_in_bytes - %s", err) } @@ -78,7 +79,7 @@ func TestMemorySetMemoryswap(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.memsw.limit_in_bytes - %s", err) } @@ -115,14 +116,14 @@ func TestMemorySetMemoryLargerThanSwap(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.limit_in_bytes - %s", err) } if value != memoryAfter { t.Fatal("Got the wrong value, set memory.limit_in_bytes failed.") } - value, err = getCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes") + value, err = fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.memsw.limit_in_bytes - %s", err) } @@ -159,14 +160,14 @@ func TestMemorySetSwapSmallerThanMemory(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.limit_in_bytes - %s", err) } if value != memoryAfter { t.Fatal("Got the wrong value, set memory.limit_in_bytes failed.") } - value, err = getCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes") + value, err = fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.memsw.limit_in_bytes - %s", err) } @@ -194,7 +195,7 @@ func TestMemorySetKernelMemory(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.kmem.limit_in_bytes") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.kmem.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.kmem.limit_in_bytes - %s", err) } @@ -222,7 +223,7 @@ func TestMemorySetKernelMemoryTCP(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.kmem.tcp.limit_in_bytes") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.kmem.tcp.limit_in_bytes") if err != nil { t.Fatalf("Failed to parse memory.kmem.tcp.limit_in_bytes - %s", err) } @@ -248,7 +249,7 @@ func TestMemorySetMemorySwappinessDefault(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.swappiness") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.swappiness") if err != nil { t.Fatalf("Failed to parse memory.swappiness - %s", err) } @@ -444,7 +445,7 @@ func TestMemorySetOomControl(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "memory.oom_control") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "memory.oom_control") if err != nil { t.Fatalf("Failed to parse memory.oom_control - %s", err) } diff --git a/libcontainer/cgroups/fs/memory_v2.go b/libcontainer/cgroups/fs/memory_v2.go deleted file mode 100644 index 2ad997bcc29..00000000000 --- a/libcontainer/cgroups/fs/memory_v2.go +++ /dev/null @@ -1,164 +0,0 @@ -// +build linux - -package fs - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/configs" -) - -type MemoryGroupV2 struct { -} - -func (s *MemoryGroupV2) Name() string { - return "memory" -} - -func (s *MemoryGroupV2) Apply(d *cgroupData) (err error) { - path, err := d.path("memory") - if err != nil && !cgroups.IsNotFound(err) { - return err - } else if path == "" { - return nil - } - if memoryAssigned(d.config) { - if _, err := os.Stat(path); os.IsNotExist(err) { - if err := os.MkdirAll(path, 0755); err != nil { - return err - } - // Only enable kernel memory accouting when this cgroup - // is created by libcontainer, otherwise we might get - // error when people use `cgroupsPath` to join an existed - // cgroup whose kernel memory is not initialized. - if err := EnableKernelMemoryAccounting(path); err != nil { - return err - } - } - } - defer func() { - if err != nil { - os.RemoveAll(path) - } - }() - - // We need to join memory cgroup after set memory limits, because - // kmem.limit_in_bytes can only be set when the cgroup is empty. - _, err = d.join("memory") - if err != nil && !cgroups.IsNotFound(err) { - return err - } - return nil -} - -func setMemoryAndSwapCgroups(path string, cgroup *configs.Cgroup) error { - if cgroup.Resources.MemorySwap != 0 { - if err := writeFile(path, "memory.swap.max", strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { - return err - } - } - if cgroup.Resources.Memory != 0 { - if err := writeFile(path, "memory.max", strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { - return err - } - } - return nil -} - -func (s *MemoryGroupV2) Set(path string, cgroup *configs.Cgroup) error { - - if err := setMemoryAndSwapCgroups(path, cgroup); err != nil { - return err - } - - if cgroup.Resources.KernelMemory != 0 { - if err := setKernelMemory(path, cgroup.Resources.KernelMemory); err != nil { - return err - } - } - - if cgroup.Resources.MemoryReservation != 0 { - if err := writeFile(path, "memory.high", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil { - return err - } - } - - return nil -} - -func (s *MemoryGroupV2) Remove(d *cgroupData) error { - return removePath(d.path("memory")) -} - -func (s *MemoryGroupV2) GetStats(path string, stats *cgroups.Stats) error { - // Set stats from memory.stat. - statsFile, err := os.Open(filepath.Join(path, "memory.stat")) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer statsFile.Close() - - sc := bufio.NewScanner(statsFile) - for sc.Scan() { - t, v, err := getCgroupParamKeyValue(sc.Text()) - if err != nil { - return fmt.Errorf("failed to parse memory.stat (%q) - %v", sc.Text(), err) - } - stats.MemoryStats.Stats[t] = v - } - stats.MemoryStats.Cache = stats.MemoryStats.Stats["cache"] - - memoryUsage, err := getMemoryDataV2(path, "") - if err != nil { - return err - } - stats.MemoryStats.Usage = memoryUsage - swapUsage, err := getMemoryDataV2(path, "swap") - if err != nil { - return err - } - stats.MemoryStats.SwapUsage = swapUsage - - stats.MemoryStats.UseHierarchy = true - return nil -} - -func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) { - memoryData := cgroups.MemoryData{} - - moduleName := "memory" - if name != "" { - moduleName = strings.Join([]string{"memory", name}, ".") - } - usage := strings.Join([]string{moduleName, "current"}, ".") - limit := strings.Join([]string{moduleName, "max"}, ".") - - value, err := getCgroupParamUint(path, usage) - if err != nil { - if moduleName != "memory" && os.IsNotExist(err) { - return cgroups.MemoryData{}, nil - } - return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", usage, err) - } - memoryData.Usage = value - - value, err = getCgroupParamUint(path, limit) - if err != nil { - if moduleName != "memory" && os.IsNotExist(err) { - return cgroups.MemoryData{}, nil - } - return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", limit, err) - } - memoryData.Limit = value - - return memoryData, nil -} diff --git a/libcontainer/cgroups/fs/net_cls.go b/libcontainer/cgroups/fs/net_cls.go index 8e74b645eac..0212015255e 100644 --- a/libcontainer/cgroups/fs/net_cls.go +++ b/libcontainer/cgroups/fs/net_cls.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -26,7 +27,7 @@ func (s *NetClsGroup) Apply(d *cgroupData) error { func (s *NetClsGroup) Set(path string, cgroup *configs.Cgroup) error { if cgroup.Resources.NetClsClassid != 0 { - if err := writeFile(path, "net_cls.classid", strconv.FormatUint(uint64(cgroup.Resources.NetClsClassid), 10)); err != nil { + if err := fscommon.WriteFile(path, "net_cls.classid", strconv.FormatUint(uint64(cgroup.Resources.NetClsClassid), 10)); err != nil { return err } } diff --git a/libcontainer/cgroups/fs/net_cls_test.go b/libcontainer/cgroups/fs/net_cls_test.go index c00e8a8d6f4..602133a265d 100644 --- a/libcontainer/cgroups/fs/net_cls_test.go +++ b/libcontainer/cgroups/fs/net_cls_test.go @@ -5,6 +5,8 @@ package fs import ( "strconv" "testing" + + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" ) const ( @@ -29,7 +31,7 @@ func TestNetClsSetClassid(t *testing.T) { // As we are in mock environment, we can't get correct value of classid from // net_cls.classid. // So. we just judge if we successfully write classid into file - value, err := getCgroupParamUint(helper.CgroupPath, "net_cls.classid") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "net_cls.classid") if err != nil { t.Fatalf("Failed to parse net_cls.classid - %s", err) } diff --git a/libcontainer/cgroups/fs/net_prio.go b/libcontainer/cgroups/fs/net_prio.go index d0ab2af894f..2bdeedf80af 100644 --- a/libcontainer/cgroups/fs/net_prio.go +++ b/libcontainer/cgroups/fs/net_prio.go @@ -4,6 +4,7 @@ package fs import ( "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -24,7 +25,7 @@ func (s *NetPrioGroup) Apply(d *cgroupData) error { func (s *NetPrioGroup) Set(path string, cgroup *configs.Cgroup) error { for _, prioMap := range cgroup.Resources.NetPrioIfpriomap { - if err := writeFile(path, "net_prio.ifpriomap", prioMap.CgroupString()); err != nil { + if err := fscommon.WriteFile(path, "net_prio.ifpriomap", prioMap.CgroupString()); err != nil { return err } } diff --git a/libcontainer/cgroups/fs/net_prio_test.go b/libcontainer/cgroups/fs/net_prio_test.go index efbf0639a0b..2ce8e1922d9 100644 --- a/libcontainer/cgroups/fs/net_prio_test.go +++ b/libcontainer/cgroups/fs/net_prio_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -28,7 +29,7 @@ func TestNetPrioSetIfPrio(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "net_prio.ifpriomap") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "net_prio.ifpriomap") if err != nil { t.Fatalf("Failed to parse net_prio.ifpriomap - %s", err) } diff --git a/libcontainer/cgroups/fs/pids.go b/libcontainer/cgroups/fs/pids.go index f1e37205518..7bf68013589 100644 --- a/libcontainer/cgroups/fs/pids.go +++ b/libcontainer/cgroups/fs/pids.go @@ -8,6 +8,7 @@ import ( "strconv" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -35,7 +36,7 @@ func (s *PidsGroup) Set(path string, cgroup *configs.Cgroup) error { limit = strconv.FormatInt(cgroup.Resources.PidsLimit, 10) } - if err := writeFile(path, "pids.max", limit); err != nil { + if err := fscommon.WriteFile(path, "pids.max", limit); err != nil { return err } } @@ -48,12 +49,12 @@ func (s *PidsGroup) Remove(d *cgroupData) error { } func (s *PidsGroup) GetStats(path string, stats *cgroups.Stats) error { - current, err := getCgroupParamUint(path, "pids.current") + current, err := fscommon.GetCgroupParamUint(path, "pids.current") if err != nil { return fmt.Errorf("failed to parse pids.current - %s", err) } - maxString, err := getCgroupParamString(path, "pids.max") + maxString, err := fscommon.GetCgroupParamString(path, "pids.max") if err != nil { return fmt.Errorf("failed to parse pids.max - %s", err) } @@ -61,7 +62,7 @@ func (s *PidsGroup) GetStats(path string, stats *cgroups.Stats) error { // Default if pids.max == "max" is 0 -- which represents "no limit". var max uint64 if maxString != "max" { - max, err = parseUint(maxString, 10, 64) + max, err = fscommon.ParseUint(maxString, 10, 64) if err != nil { return fmt.Errorf("failed to parse pids.max - unable to parse %q as a uint from Cgroup file %q", maxString, filepath.Join(path, "pids.max")) } diff --git a/libcontainer/cgroups/fs/pids_test.go b/libcontainer/cgroups/fs/pids_test.go index 10671247ba3..66f3aa3315e 100644 --- a/libcontainer/cgroups/fs/pids_test.go +++ b/libcontainer/cgroups/fs/pids_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" ) const ( @@ -28,7 +29,7 @@ func TestPidsSetMax(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamUint(helper.CgroupPath, "pids.max") + value, err := fscommon.GetCgroupParamUint(helper.CgroupPath, "pids.max") if err != nil { t.Fatalf("Failed to parse pids.max - %s", err) } @@ -52,7 +53,7 @@ func TestPidsSetUnlimited(t *testing.T) { t.Fatal(err) } - value, err := getCgroupParamString(helper.CgroupPath, "pids.max") + value, err := fscommon.GetCgroupParamString(helper.CgroupPath, "pids.max") if err != nil { t.Fatalf("Failed to parse pids.max - %s", err) } diff --git a/libcontainer/cgroups/fs/pids_v2.go b/libcontainer/cgroups/fs/pids_v2.go deleted file mode 100644 index 3413a2a0d18..00000000000 --- a/libcontainer/cgroups/fs/pids_v2.go +++ /dev/null @@ -1,107 +0,0 @@ -// +build linux - -package fs - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/configs" - "golang.org/x/sys/unix" -) - -type PidsGroupV2 struct { -} - -func (s *PidsGroupV2) Name() string { - return "pids" -} - -func (s *PidsGroupV2) Apply(d *cgroupData) error { - _, err := d.join("pids") - if err != nil && !cgroups.IsNotFound(err) { - return err - } - return nil -} - -func (s *PidsGroupV2) Set(path string, cgroup *configs.Cgroup) error { - if cgroup.Resources.PidsLimit != 0 { - // "max" is the fallback value. - limit := "max" - - if cgroup.Resources.PidsLimit > 0 { - limit = strconv.FormatInt(cgroup.Resources.PidsLimit, 10) - } - - if err := writeFile(path, "pids.max", limit); err != nil { - return err - } - } - - return nil -} - -func (s *PidsGroupV2) Remove(d *cgroupData) error { - return removePath(d.path("pids")) -} - -func isNOTSUP(err error) bool { - switch err := err.(type) { - case *os.PathError: - return err.Err == unix.ENOTSUP - default: - return false - } -} - -func (s *PidsGroupV2) GetStats(path string, stats *cgroups.Stats) error { - current, err := getCgroupParamUint(path, "pids.current") - if os.IsNotExist(err) { - // if the controller is not enabled, let's read the list - // PIDs (or threads if cgroup.threads is enabled) - contents, err := ioutil.ReadFile(filepath.Join(path, "cgroup.procs")) - if err != nil && isNOTSUP(err) { - contents, err = ioutil.ReadFile(filepath.Join(path, "cgroup.threads")) - } - if err != nil { - return err - } - pids := make(map[string]string) - for _, i := range strings.Split(string(contents), "\n") { - if i != "" { - pids[i] = i - } - } - stats.PidsStats.Current = uint64(len(pids)) - stats.PidsStats.Limit = 0 - return nil - - } - if err != nil { - return fmt.Errorf("failed to parse pids.current - %s", err) - } - - maxString, err := getCgroupParamString(path, "pids.max") - if err != nil { - return fmt.Errorf("failed to parse pids.max - %s", err) - } - - // Default if pids.max == "max" is 0 -- which represents "no limit". - var max uint64 - if maxString != "max" { - max, err = parseUint(maxString, 10, 64) - if err != nil { - return fmt.Errorf("failed to parse pids.max - unable to parse %q as a uint from Cgroup file %q", maxString, filepath.Join(path, "pids.max")) - } - } - - stats.PidsStats.Current = current - stats.PidsStats.Limit = max - return nil -} diff --git a/libcontainer/cgroups/fs/util_test.go b/libcontainer/cgroups/fs/util_test.go index 7067e799fbe..2c50d6f3a29 100644 --- a/libcontainer/cgroups/fs/util_test.go +++ b/libcontainer/cgroups/fs/util_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "testing" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -59,7 +60,7 @@ func (c *cgroupTestUtil) cleanup() { // Write the specified contents on the mock of the specified cgroup files. func (c *cgroupTestUtil) writeFileContents(fileContents map[string]string) { for file, contents := range fileContents { - err := writeFile(c.CgroupPath, file, contents) + err := fscommon.WriteFile(c.CgroupPath, file, contents) if err != nil { c.t.Fatal(err) } diff --git a/libcontainer/cgroups/fs2/cpu.go b/libcontainer/cgroups/fs2/cpu.go new file mode 100644 index 00000000000..f0f5df09a80 --- /dev/null +++ b/libcontainer/cgroups/fs2/cpu.go @@ -0,0 +1,56 @@ +// +build linux + +package fs2 + +import ( + "bufio" + "os" + "path/filepath" + "strconv" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" + "github.com/opencontainers/runc/libcontainer/configs" +) + +func setCpu(dirPath string, cgroup *configs.Cgroup) error { + if cgroup.Resources.CpuWeight != 0 { + if err := fscommon.WriteFile(dirPath, "cpu.weight", strconv.FormatUint(cgroup.Resources.CpuWeight, 10)); err != nil { + return err + } + } + + if cgroup.Resources.CpuMax != "" { + if err := fscommon.WriteFile(dirPath, "cpu.max", cgroup.Resources.CpuMax); err != nil { + return err + } + } + + return nil +} +func statCpu(dirPath string, stats *cgroups.Stats) error { + f, err := os.Open(filepath.Join(dirPath, "cpu.stat")) + if err != nil { + return err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + t, v, err := fscommon.GetCgroupParamKeyValue(sc.Text()) + if err != nil { + return err + } + switch t { + case "usage_usec": + stats.CpuStats.CpuUsage.TotalUsage = v * 1000 + + case "user_usec": + stats.CpuStats.CpuUsage.UsageInUsermode = v * 1000 + + case "system_usec": + stats.CpuStats.CpuUsage.UsageInKernelmode = v * 1000 + } + } + return nil +} diff --git a/libcontainer/cgroups/fs2/cpuset.go b/libcontainer/cgroups/fs2/cpuset.go new file mode 100644 index 00000000000..6492ac93c58 --- /dev/null +++ b/libcontainer/cgroups/fs2/cpuset.go @@ -0,0 +1,22 @@ +// +build linux + +package fs2 + +import ( + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" + "github.com/opencontainers/runc/libcontainer/configs" +) + +func setCpuset(dirPath string, cgroup *configs.Cgroup) error { + if cgroup.Resources.CpusetCpus != "" { + if err := fscommon.WriteFile(dirPath, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil { + return err + } + } + if cgroup.Resources.CpusetMems != "" { + if err := fscommon.WriteFile(dirPath, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil { + return err + } + } + return nil +} diff --git a/libcontainer/cgroups/fs2/defaultpath.go b/libcontainer/cgroups/fs2/defaultpath.go new file mode 100644 index 00000000000..e84b33f2c70 --- /dev/null +++ b/libcontainer/cgroups/fs2/defaultpath.go @@ -0,0 +1,99 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fs2 + +import ( + "bufio" + "io" + "os" + "path/filepath" + "strings" + + "github.com/opencontainers/runc/libcontainer/configs" + libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" + "github.com/pkg/errors" +) + +const UnifiedMountpoint = "/sys/fs/cgroup" + +func defaultDirPath(c *configs.Cgroup) (string, error) { + if (c.Name != "" || c.Parent != "") && c.Path != "" { + return "", errors.Errorf("cgroup: either Path or Name and Parent should be used, got %+v", c) + } + if len(c.Paths) != 0 { + // never set by specconv + return "", errors.Errorf("cgroup: Paths is unsupported, use Path, got %+v", c) + } + + // XXX: Do not remove this code. Path safety is important! -- cyphar + cgPath := libcontainerUtils.CleanPath(c.Path) + cgParent := libcontainerUtils.CleanPath(c.Parent) + cgName := libcontainerUtils.CleanPath(c.Name) + + ownCgroup, err := parseCgroupFile("/proc/self/cgroup") + if err != nil { + return "", err + } + return _defaultDirPath(UnifiedMountpoint, cgPath, cgParent, cgName, ownCgroup) +} + +func _defaultDirPath(root, cgPath, cgParent, cgName, ownCgroup string) (string, error) { + if (cgName != "" || cgParent != "") && cgPath != "" { + return "", errors.New("cgroup: either Path or Name and Parent should be used") + } + innerPath := cgPath + if innerPath == "" { + innerPath = filepath.Join(cgParent, cgName) + } + if filepath.IsAbs(innerPath) { + return filepath.Join(root, innerPath), nil + } + return filepath.Join(root, ownCgroup, innerPath), nil +} + +// parseCgroupFile parses /proc/PID/cgroup file and return string +func parseCgroupFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + return parseCgroupFromReader(f) +} + +func parseCgroupFromReader(r io.Reader) (string, error) { + var ( + s = bufio.NewScanner(r) + ) + for s.Scan() { + if err := s.Err(); err != nil { + return "", err + } + var ( + text = s.Text() + parts = strings.SplitN(text, ":", 3) + ) + if len(parts) < 3 { + return "", errors.Errorf("invalid cgroup entry: %q", text) + } + // text is like "0::/user.slice/user-1001.slice/session-1.scope" + if parts[0] == "0" && parts[1] == "" { + return parts[2], nil + } + } + return "", errors.New("cgroup path not found") +} diff --git a/libcontainer/cgroups/fs2/defaultpath_test.go b/libcontainer/cgroups/fs2/defaultpath_test.go new file mode 100644 index 00000000000..6d5d1170800 --- /dev/null +++ b/libcontainer/cgroups/fs2/defaultpath_test.go @@ -0,0 +1,76 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fs2 + +import ( + "strings" + "testing" +) + +func TestParseCgroupFromReader(t *testing.T) { + cases := map[string]string{ + "0::/user.slice/user-1001.slice/session-1.scope\n": "/user.slice/user-1001.slice/session-1.scope", + "2:cpuset:/foo\n1:name=systemd:/\n": "", + "2:cpuset:/foo\n1:name=systemd:/\n0::/user.slice/user-1001.slice/session-1.scope\n": "/user.slice/user-1001.slice/session-1.scope", + } + for s, expected := range cases { + g, err := parseCgroupFromReader(strings.NewReader(s)) + if expected != "" { + if string(g) != expected { + t.Errorf("expected %q, got %q", expected, string(g)) + } + if err != nil { + t.Error(err) + } + } else { + if err == nil { + t.Error("error is expected") + } + } + } +} + +func TestDefaultDirPath(t *testing.T) { + root := "/sys/fs/cgroup" + cases := []struct { + cgPath string + cgParent string + cgName string + ownCgroup string + expected string + }{ + { + cgPath: "/foo/bar", + ownCgroup: "/apple/banana", + expected: "/sys/fs/cgroup/foo/bar", + }, + { + cgPath: "foo/bar", + ownCgroup: "/apple/banana", + expected: "/sys/fs/cgroup/apple/banana/foo/bar", + }, + } + for _, c := range cases { + got, err := _defaultDirPath(root, c.cgPath, c.cgParent, c.cgName, c.ownCgroup) + if err != nil { + t.Fatal(err) + } + if got != c.expected { + t.Fatalf("expected %q, got %q", c.expected, got) + } + } +} diff --git a/libcontainer/cgroups/fs/devices_v2.go b/libcontainer/cgroups/fs2/devices.go similarity index 70% rename from libcontainer/cgroups/fs/devices_v2.go rename to libcontainer/cgroups/fs2/devices.go index 98512539e5d..f852b53ede2 100644 --- a/libcontainer/cgroups/fs/devices_v2.go +++ b/libcontainer/cgroups/fs2/devices.go @@ -1,9 +1,8 @@ // +build linux -package fs +package fs2 import ( - "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/ebpf" "github.com/opencontainers/runc/libcontainer/cgroups/ebpf/devicefilter" "github.com/opencontainers/runc/libcontainer/configs" @@ -11,17 +10,6 @@ import ( "golang.org/x/sys/unix" ) -type DevicesGroupV2 struct { -} - -func (s *DevicesGroupV2) Name() string { - return "devices" -} - -func (s *DevicesGroupV2) Apply(d *cgroupData) error { - return nil -} - func isRWM(cgroupPermissions string) bool { r := false w := false @@ -50,7 +38,7 @@ func canSkipEBPFError(cgroup *configs.Cgroup) bool { return true } -func (s *DevicesGroupV2) Set(path string, cgroup *configs.Cgroup) error { +func setDevices(dirPath string, cgroup *configs.Cgroup) error { if cgroup.Resources.AllowAllDevices != nil { // never set by OCI specconv return errors.New("libcontainer AllowAllDevices is not supported, use Devices") @@ -63,9 +51,9 @@ func (s *DevicesGroupV2) Set(path string, cgroup *configs.Cgroup) error { if err != nil { return err } - dirFD, err := unix.Open(path, unix.O_DIRECTORY|unix.O_RDONLY, 0600) + dirFD, err := unix.Open(dirPath, unix.O_DIRECTORY|unix.O_RDONLY, 0600) if err != nil { - return errors.Errorf("cannot get dir FD for %s", path) + return errors.Errorf("cannot get dir FD for %s", dirPath) } defer unix.Close(dirFD) if _, err := ebpf.LoadAttachCgroupDeviceFilter(insts, license, dirFD); err != nil { @@ -75,11 +63,3 @@ func (s *DevicesGroupV2) Set(path string, cgroup *configs.Cgroup) error { } return nil } - -func (s *DevicesGroupV2) Remove(d *cgroupData) error { - return nil -} - -func (s *DevicesGroupV2) GetStats(path string, stats *cgroups.Stats) error { - return nil -} diff --git a/libcontainer/cgroups/fs2/freezer.go b/libcontainer/cgroups/fs2/freezer.go new file mode 100644 index 00000000000..130c63f3e2b --- /dev/null +++ b/libcontainer/cgroups/fs2/freezer.go @@ -0,0 +1,53 @@ +// +build linux + +package fs2 + +import ( + "strconv" + "strings" + + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/pkg/errors" +) + +func setFreezer(dirPath string, state configs.FreezerState) error { + var desired int + switch state { + case configs.Undefined: + return nil + case configs.Frozen: + desired = 1 + case configs.Thawed: + desired = 0 + default: + return errors.Errorf("unknown freezer state %+v", state) + } + supportedErr := supportsFreezer(dirPath) + if supportedErr != nil && desired != 0 { + // can ignore error if desired == 1 + return errors.Wrap(supportedErr, "freezer not supported") + } + return freezeWithInt(dirPath, desired) +} + +func supportsFreezer(dirPath string) error { + _, err := fscommon.ReadFile(dirPath, "cgroup.freeze") + return err +} + +// freeze writes desired int to "cgroup.freeze". +func freezeWithInt(dirPath string, desired int) error { + desiredS := strconv.Itoa(desired) + if err := fscommon.WriteFile(dirPath, "cgroup.freeze", desiredS); err != nil { + return err + } + got, err := fscommon.ReadFile(dirPath, "cgroup.freeze") + if err != nil { + return err + } + if gotS := strings.TrimSpace(string(got)); gotS != desiredS { + return errors.Errorf("expected \"cgroup.freeze\" in %q to be %q, got %q", dirPath, desiredS, gotS) + } + return nil +} diff --git a/libcontainer/cgroups/fs2/fs2.go b/libcontainer/cgroups/fs2/fs2.go new file mode 100644 index 00000000000..2d32f8210e1 --- /dev/null +++ b/libcontainer/cgroups/fs2/fs2.go @@ -0,0 +1,209 @@ +// +build linux + +package fs2 + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/pkg/errors" +) + +// NewManager creates a manager for cgroup v2 unified hierarchy. +// dirPath is like "/sys/fs/cgroup/user.slice/user-1001.slice/session-1.scope". +// If dirPath is empty, it is automatically set using config. +func NewManager(config *configs.Cgroup, dirPath string, rootless bool) (cgroups.Manager, error) { + if config == nil { + config = &configs.Cgroup{} + } + if dirPath != "" { + if filepath.Clean(dirPath) != dirPath || !filepath.IsAbs(dirPath) { + return nil, errors.Errorf("invalid dir path %q", dirPath) + } + } else { + var err error + dirPath, err = defaultDirPath(config) + if err != nil { + return nil, err + } + } + controllers, err := detectControllers(dirPath) + if err != nil && !rootless { + return nil, err + } + + m := &manager{ + config: config, + dirPath: dirPath, + controllers: controllers, + rootless: rootless, + } + return m, nil +} + +func detectControllers(dirPath string) (map[string]struct{}, error) { + if err := os.MkdirAll(dirPath, 0755); err != nil { + return nil, err + } + controllersPath, err := securejoin.SecureJoin(dirPath, "cgroup.controllers") + if err != nil { + return nil, err + } + controllersData, err := ioutil.ReadFile(controllersPath) + if err != nil { + return nil, err + } + controllersFields := strings.Fields(string(controllersData)) + controllers := make(map[string]struct{}, len(controllersFields)) + for _, c := range controllersFields { + controllers[c] = struct{}{} + } + return controllers, nil +} + +type manager struct { + config *configs.Cgroup + // dirPath is like "/sys/fs/cgroup/user.slice/user-1001.slice/session-1.scope" + dirPath string + // controllers is content of "cgroup.controllers" file. + // excludes pseudo-controllers ("devices" and "freezer"). + controllers map[string]struct{} + rootless bool +} + +func (m *manager) Apply(pid int) error { + if err := cgroups.WriteCgroupProc(m.dirPath, pid); err != nil && !m.rootless { + return err + } + return nil +} + +func (m *manager) GetPids() ([]int, error) { + return cgroups.GetPids(m.dirPath) +} + +func (m *manager) GetAllPids() ([]int, error) { + return cgroups.GetAllPids(m.dirPath) +} + +func (m *manager) GetStats() (*cgroups.Stats, error) { + var ( + st cgroups.Stats + errs []error + ) + // pids (since kernel 4.5) + if _, ok := m.controllers["pids"]; ok { + if err := statPids(m.dirPath, &st); err != nil { + errs = append(errs, err) + } + } else { + if err := statPidsWithoutController(m.dirPath, &st); err != nil { + errs = append(errs, err) + } + } + // memory (since kenrel 4.5) + if _, ok := m.controllers["memory"]; ok { + if err := statMemory(m.dirPath, &st); err != nil { + errs = append(errs, err) + } + } + // io (since kernel 4.5) + if _, ok := m.controllers["io"]; ok { + if err := statIo(m.dirPath, &st); err != nil { + errs = append(errs, err) + } + } + // cpu (since kernel 4.15) + if _, ok := m.controllers["cpu"]; ok { + if err := statCpu(m.dirPath, &st); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 && !m.rootless { + return &st, errors.Errorf("error while statting cgroup v2: %+v", errs) + } + return &st, nil +} + +func (m *manager) Freeze(state configs.FreezerState) error { + if err := setFreezer(m.dirPath, state); err != nil { + return err + } + m.config.Resources.Freezer = state + return nil +} + +func (m *manager) Destroy() error { + return os.RemoveAll(m.dirPath) +} + +// GetPaths is for compatibility purpose and should be removed in future +func (m *manager) GetPaths() map[string]string { + paths := map[string]string{ + // pseudo-controller for compatibility + "devices": m.dirPath, + "freezer": m.dirPath, + } + for c := range m.controllers { + paths[c] = m.dirPath + } + return paths +} + +func (m *manager) GetUnifiedPath() (string, error) { + return m.dirPath, nil +} + +func (m *manager) Set(container *configs.Config) error { + if container == nil || container.Cgroups == nil { + return nil + } + var errs []error + // pids (since kernel 4.5) + if _, ok := m.controllers["pids"]; ok { + if err := setPids(m.dirPath, container.Cgroups); err != nil { + errs = append(errs, err) + } + } + // memory (since kernel 4.5) + if _, ok := m.controllers["memory"]; ok { + if err := setMemory(m.dirPath, container.Cgroups); err != nil { + errs = append(errs, err) + } + } + // io (since kernel 4.5) + if _, ok := m.controllers["io"]; ok { + if err := setIo(m.dirPath, container.Cgroups); err != nil { + errs = append(errs, err) + } + } + // cpu (since kernel 4.15) + if _, ok := m.controllers["cpu"]; ok { + if err := setCpu(m.dirPath, container.Cgroups); err != nil { + errs = append(errs, err) + } + } + // devices (since kernel 4.15, pseudo-controller) + if err := setDevices(m.dirPath, container.Cgroups); err != nil { + errs = append(errs, err) + } + // cpuset (since kernel 5.0) + if _, ok := m.controllers["cpuset"]; ok { + if err := setCpuset(m.dirPath, container.Cgroups); err != nil { + errs = append(errs, err) + } + } + // freezer (since kernel 5.2, pseudo-controller) + if err := setFreezer(m.dirPath, container.Cgroups.Freezer); err != nil { + errs = append(errs, err) + } + if len(errs) > 0 && !m.rootless { + return errors.Errorf("error while setting cgroup v2: %+v", errs) + } + return nil +} diff --git a/libcontainer/cgroups/fs/io_v2.go b/libcontainer/cgroups/fs2/io.go similarity index 62% rename from libcontainer/cgroups/fs/io_v2.go rename to libcontainer/cgroups/fs2/io.go index 477f7325b19..9a07308802a 100644 --- a/libcontainer/cgroups/fs/io_v2.go +++ b/libcontainer/cgroups/fs2/io.go @@ -1,6 +1,6 @@ // +build linux -package fs +package fs2 import ( "bufio" @@ -10,49 +10,35 @@ import ( "strings" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" "github.com/opencontainers/runc/libcontainer/configs" ) -type IOGroupV2 struct { -} - -func (s *IOGroupV2) Name() string { - return "io" -} - -func (s *IOGroupV2) Apply(d *cgroupData) error { - _, err := d.join("io") - if err != nil && !cgroups.IsNotFound(err) { - return err - } - return nil -} - -func (s *IOGroupV2) Set(path string, cgroup *configs.Cgroup) error { +func setIo(dirPath string, cgroup *configs.Cgroup) error { if cgroup.Resources.BlkioWeight != 0 { filename := "io.bfq.weight" - if err := writeFile(path, filename, strconv.FormatUint(uint64(cgroup.Resources.BlkioWeight), 10)); err != nil { + if err := fscommon.WriteFile(dirPath, filename, strconv.FormatUint(uint64(cgroup.Resources.BlkioWeight), 10)); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleReadBpsDevice { - if err := writeFile(path, "io.max", td.StringName("rbps")); err != nil { + if err := fscommon.WriteFile(dirPath, "io.max", td.StringName("rbps")); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleWriteBpsDevice { - if err := writeFile(path, "io.max", td.StringName("wbps")); err != nil { + if err := fscommon.WriteFile(dirPath, "io.max", td.StringName("wbps")); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleReadIOPSDevice { - if err := writeFile(path, "io.max", td.StringName("riops")); err != nil { + if err := fscommon.WriteFile(dirPath, "io.max", td.StringName("riops")); err != nil { return err } } for _, td := range cgroup.Resources.BlkioThrottleWriteIOPSDevice { - if err := writeFile(path, "io.max", td.StringName("wiops")); err != nil { + if err := fscommon.WriteFile(dirPath, "io.max", td.StringName("wiops")); err != nil { return err } } @@ -60,18 +46,11 @@ func (s *IOGroupV2) Set(path string, cgroup *configs.Cgroup) error { return nil } -func (s *IOGroupV2) Remove(d *cgroupData) error { - return removePath(d.path("io")) -} - -func readCgroup2MapFile(path string, name string) (map[string][]string, error) { +func readCgroup2MapFile(dirPath string, name string) (map[string][]string, error) { ret := map[string][]string{} - p := filepath.Join("/sys/fs/cgroup", path, name) + p := filepath.Join(dirPath, name) f, err := os.Open(p) if err != nil { - if os.IsNotExist(err) { - return ret, nil - } return nil, err } defer f.Close() @@ -90,10 +69,10 @@ func readCgroup2MapFile(path string, name string) (map[string][]string, error) { return ret, nil } -func (s *IOGroupV2) getCgroupV2Stats(path string, stats *cgroups.Stats) error { +func statIo(dirPath string, stats *cgroups.Stats) error { // more details on the io.stat file format: https://www.kernel.org/doc/Documentation/cgroup-v2.txt var ioServiceBytesRecursive []cgroups.BlkioStatEntry - values, err := readCgroup2MapFile(path, "io.stat") + values, err := readCgroup2MapFile(dirPath, "io.stat") if err != nil { return err } @@ -143,7 +122,3 @@ func (s *IOGroupV2) getCgroupV2Stats(path string, stats *cgroups.Stats) error { stats.BlkioStats = cgroups.BlkioStats{IoServiceBytesRecursive: ioServiceBytesRecursive} return nil } - -func (s *IOGroupV2) GetStats(path string, stats *cgroups.Stats) error { - return s.getCgroupV2Stats(path, stats) -} diff --git a/libcontainer/cgroups/fs2/memory.go b/libcontainer/cgroups/fs2/memory.go new file mode 100644 index 00000000000..23eccbec40d --- /dev/null +++ b/libcontainer/cgroups/fs2/memory.go @@ -0,0 +1,103 @@ +// +build linux + +package fs2 + +import ( + "bufio" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/pkg/errors" +) + +func setMemory(dirPath string, cgroup *configs.Cgroup) error { + if cgroup.Resources.MemorySwap != 0 { + if err := fscommon.WriteFile(dirPath, "memory.swap.max", strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil { + return err + } + } + if cgroup.Resources.Memory != 0 { + if err := fscommon.WriteFile(dirPath, "memory.max", strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil { + return err + } + } + + // cgroup.Resources.KernelMemory is ignored + + if cgroup.Resources.MemoryReservation != 0 { + if err := fscommon.WriteFile(dirPath, "memory.low", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil { + return err + } + } + + return nil +} + +func statMemory(dirPath string, stats *cgroups.Stats) error { + // Set stats from memory.stat. + statsFile, err := os.Open(filepath.Join(dirPath, "memory.stat")) + if err != nil { + return err + } + defer statsFile.Close() + + sc := bufio.NewScanner(statsFile) + for sc.Scan() { + t, v, err := fscommon.GetCgroupParamKeyValue(sc.Text()) + if err != nil { + return errors.Wrapf(err, "failed to parse memory.stat (%q)", sc.Text()) + } + stats.MemoryStats.Stats[t] = v + } + stats.MemoryStats.Cache = stats.MemoryStats.Stats["cache"] + + memoryUsage, err := getMemoryDataV2(dirPath, "") + if err != nil { + return err + } + stats.MemoryStats.Usage = memoryUsage + swapUsage, err := getMemoryDataV2(dirPath, "swap") + if err != nil { + return err + } + stats.MemoryStats.SwapUsage = swapUsage + + stats.MemoryStats.UseHierarchy = true + return nil +} + +func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) { + memoryData := cgroups.MemoryData{} + + moduleName := "memory" + if name != "" { + moduleName = strings.Join([]string{"memory", name}, ".") + } + usage := strings.Join([]string{moduleName, "current"}, ".") + limit := strings.Join([]string{moduleName, "max"}, ".") + + value, err := fscommon.GetCgroupParamUint(path, usage) + if err != nil { + if moduleName != "memory" && os.IsNotExist(err) { + return cgroups.MemoryData{}, nil + } + return cgroups.MemoryData{}, errors.Wrapf(err, "failed to parse %s", usage) + } + memoryData.Usage = value + + value, err = fscommon.GetCgroupParamUint(path, limit) + if err != nil { + if moduleName != "memory" && os.IsNotExist(err) { + return cgroups.MemoryData{}, nil + } + return cgroups.MemoryData{}, errors.Wrapf(err, "failed to parse %s", limit) + } + memoryData.Limit = value + + return memoryData, nil +} diff --git a/libcontainer/cgroups/fs2/pids.go b/libcontainer/cgroups/fs2/pids.go new file mode 100644 index 00000000000..db2d7ac90da --- /dev/null +++ b/libcontainer/cgroups/fs2/pids.go @@ -0,0 +1,90 @@ +// +build linux + +package fs2 + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups/fscommon" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func setPids(dirPath string, cgroup *configs.Cgroup) error { + if cgroup.Resources.PidsLimit != 0 { + // "max" is the fallback value. + limit := "max" + + if cgroup.Resources.PidsLimit > 0 { + limit = strconv.FormatInt(cgroup.Resources.PidsLimit, 10) + } + + if err := fscommon.WriteFile(dirPath, "pids.max", limit); err != nil { + return err + } + } + + return nil +} + +func isNOTSUP(err error) bool { + switch err := err.(type) { + case *os.PathError: + return err.Err == unix.ENOTSUP + default: + return false + } +} + +func statPidsWithoutController(dirPath string, stats *cgroups.Stats) error { + // if the controller is not enabled, let's read PIDS from cgroups.procs + // (or threads if cgroup.threads is enabled) + contents, err := ioutil.ReadFile(filepath.Join(dirPath, "cgroup.procs")) + if err != nil && isNOTSUP(err) { + contents, err = ioutil.ReadFile(filepath.Join(dirPath, "cgroup.threads")) + } + if err != nil { + return err + } + pids := make(map[string]string) + for _, i := range strings.Split(string(contents), "\n") { + if i != "" { + pids[i] = i + } + } + stats.PidsStats.Current = uint64(len(pids)) + stats.PidsStats.Limit = 0 + return nil +} + +func statPids(dirPath string, stats *cgroups.Stats) error { + current, err := fscommon.GetCgroupParamUint(dirPath, "pids.current") + if err != nil { + return errors.Wrap(err, "failed to parse pids.current") + } + + maxString, err := fscommon.GetCgroupParamString(dirPath, "pids.max") + if err != nil { + return errors.Wrap(err, "failed to parse pids.max") + } + + // Default if pids.max == "max" is 0 -- which represents "no limit". + var max uint64 + if maxString != "max" { + max, err = fscommon.ParseUint(maxString, 10, 64) + if err != nil { + return errors.Wrapf(err, "failed to parse pids.max - unable to parse %q as a uint from Cgroup file %q", + maxString, filepath.Join(dirPath, "pids.max")) + } + } + + stats.PidsStats.Current = current + stats.PidsStats.Limit = max + return nil +} diff --git a/libcontainer/cgroups/fscommon/fscommon.go b/libcontainer/cgroups/fscommon/fscommon.go new file mode 100644 index 00000000000..dd92e8c8c67 --- /dev/null +++ b/libcontainer/cgroups/fscommon/fscommon.go @@ -0,0 +1,36 @@ +// +build linux + +package fscommon + +import ( + "io/ioutil" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/pkg/errors" +) + +func WriteFile(dir, file, data string) error { + if dir == "" { + return errors.Errorf("no directory specified for %s", file) + } + path, err := securejoin.SecureJoin(dir, file) + if err != nil { + return err + } + if err := ioutil.WriteFile(path, []byte(data), 0700); err != nil { + return errors.Wrapf(err, "failed to write %q to %q", data, path) + } + return nil +} + +func ReadFile(dir, file string) (string, error) { + if dir == "" { + return "", errors.Errorf("no directory specified for %s", file) + } + path, err := securejoin.SecureJoin(dir, file) + if err != nil { + return "", err + } + data, err := ioutil.ReadFile(path) + return string(data), err +} diff --git a/libcontainer/cgroups/fs/utils.go b/libcontainer/cgroups/fscommon/utils.go similarity index 83% rename from libcontainer/cgroups/fs/utils.go rename to libcontainer/cgroups/fscommon/utils.go index 30922777f53..46c3c7799ee 100644 --- a/libcontainer/cgroups/fs/utils.go +++ b/libcontainer/cgroups/fscommon/utils.go @@ -1,6 +1,6 @@ // +build linux -package fs +package fscommon import ( "errors" @@ -18,7 +18,7 @@ var ( // Saturates negative values at zero and returns a uint64. // Due to kernel bugs, some of the memory cgroup stats can be negative. -func parseUint(s string, base, bitSize int) (uint64, error) { +func ParseUint(s string, base, bitSize int) (uint64, error) { value, err := strconv.ParseUint(s, base, bitSize) if err != nil { intValue, intErr := strconv.ParseInt(s, base, bitSize) @@ -38,11 +38,11 @@ func parseUint(s string, base, bitSize int) (uint64, error) { // Parses a cgroup param and returns as name, value // i.e. "io_service_bytes 1234" will return as io_service_bytes, 1234 -func getCgroupParamKeyValue(t string) (string, uint64, error) { +func GetCgroupParamKeyValue(t string) (string, uint64, error) { parts := strings.Fields(t) switch len(parts) { case 2: - value, err := parseUint(parts[1], 10, 64) + value, err := ParseUint(parts[1], 10, 64) if err != nil { return "", 0, fmt.Errorf("unable to convert param value (%q) to uint64: %v", parts[1], err) } @@ -54,7 +54,7 @@ func getCgroupParamKeyValue(t string) (string, uint64, error) { } // Gets a single uint64 value from the specified cgroup file. -func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { +func GetCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { fileName := filepath.Join(cgroupPath, cgroupFile) contents, err := ioutil.ReadFile(fileName) if err != nil { @@ -65,7 +65,7 @@ func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { return math.MaxUint64, nil } - res, err := parseUint(trimmed, 10, 64) + res, err := ParseUint(trimmed, 10, 64) if err != nil { return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName) } @@ -73,7 +73,7 @@ func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { } // Gets a string value from the specified cgroup file -func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) { +func GetCgroupParamString(cgroupPath, cgroupFile string) (string, error) { contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile)) if err != nil { return "", err diff --git a/libcontainer/cgroups/fs/utils_test.go b/libcontainer/cgroups/fscommon/utils_test.go similarity index 84% rename from libcontainer/cgroups/fs/utils_test.go rename to libcontainer/cgroups/fscommon/utils_test.go index 99cdc18e071..d0c5668b5ea 100644 --- a/libcontainer/cgroups/fs/utils_test.go +++ b/libcontainer/cgroups/fscommon/utils_test.go @@ -1,6 +1,6 @@ // +build linux -package fs +package fscommon import ( "io/ioutil" @@ -31,7 +31,7 @@ func TestGetCgroupParamsInt(t *testing.T) { if err != nil { t.Fatal(err) } - value, err := getCgroupParamUint(tempDir, cgroupFile) + value, err := GetCgroupParamUint(tempDir, cgroupFile) if err != nil { t.Fatal(err) } else if value != floatValue { @@ -43,7 +43,7 @@ func TestGetCgroupParamsInt(t *testing.T) { if err != nil { t.Fatal(err) } - value, err = getCgroupParamUint(tempDir, cgroupFile) + value, err = GetCgroupParamUint(tempDir, cgroupFile) if err != nil { t.Fatal(err) } else if value != floatValue { @@ -55,7 +55,7 @@ func TestGetCgroupParamsInt(t *testing.T) { if err != nil { t.Fatal(err) } - value, err = getCgroupParamUint(tempDir, cgroupFile) + value, err = GetCgroupParamUint(tempDir, cgroupFile) if err != nil { t.Fatal(err) } else if value != 0 { @@ -68,7 +68,7 @@ func TestGetCgroupParamsInt(t *testing.T) { if err != nil { t.Fatal(err) } - value, err = getCgroupParamUint(tempDir, cgroupFile) + value, err = GetCgroupParamUint(tempDir, cgroupFile) if err != nil { t.Fatal(err) } else if value != 0 { @@ -80,7 +80,7 @@ func TestGetCgroupParamsInt(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = getCgroupParamUint(tempDir, cgroupFile) + _, err = GetCgroupParamUint(tempDir, cgroupFile) if err == nil { t.Fatal("Expecting error, got none") } @@ -90,7 +90,7 @@ func TestGetCgroupParamsInt(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = getCgroupParamUint(tempDir, cgroupFile) + _, err = GetCgroupParamUint(tempDir, cgroupFile) if err == nil { t.Fatal("Expecting error, got none") } diff --git a/libcontainer/cgroups/systemd/unified_hierarchy.go b/libcontainer/cgroups/systemd/unified_hierarchy.go index cdee5767aec..912e43b7d09 100644 --- a/libcontainer/cgroups/systemd/unified_hierarchy.go +++ b/libcontainer/cgroups/systemd/unified_hierarchy.go @@ -14,7 +14,7 @@ import ( systemdDbus "github.com/coreos/go-systemd/dbus" "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/cgroups/fs" + "github.com/opencontainers/runc/libcontainer/cgroups/fs2" "github.com/opencontainers/runc/libcontainer/configs" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -26,16 +26,6 @@ type UnifiedManager struct { Paths map[string]string } -var unifiedSubsystems = subsystemSet{ - &fs.CpusetGroupV2{}, - &fs.FreezerGroupV2{}, - &fs.CpuGroupV2{}, - &fs.MemoryGroupV2{}, - &fs.IOGroupV2{}, - &fs.PidsGroupV2{}, - &fs.DevicesGroupV2{}, -} - func (m *UnifiedManager) Apply(pid int) error { var ( c = m.Cgroups @@ -161,19 +151,19 @@ func (m *UnifiedManager) Apply(pid int) error { return err } - paths := make(map[string]string) - for _, s := range unifiedSubsystems { - subsystemPath, err := getSubsystemPath(m.Cgroups, s.Name()) - if err != nil { - // Don't fail if a cgroup hierarchy was not found, just skip this subsystem - if cgroups.IsNotFound(err) { - continue - } - return err - } - paths[s.Name()] = subsystemPath + path, err := getSubsystemPath(m.Cgroups, "") + if err != nil { + return err + } + m.Paths = map[string]string{ + "pids": path, + "memory": path, + "io": path, + "cpu": path, + "devices": path, + "cpuset": path, + "freezer": path, } - m.Paths = paths return nil } @@ -269,23 +259,20 @@ func joinCgroupsV2(c *configs.Cgroup, pid int) error { return createCgroupsv2Path(path) } -func (m *UnifiedManager) Freeze(state configs.FreezerState) error { - path, err := getSubsystemPath(m.Cgroups, "freezer") - if err != nil { - return err - } - prevState := m.Cgroups.Resources.Freezer - m.Cgroups.Resources.Freezer = state - freezer, err := unifiedSubsystems.Get("freezer") +func (m *UnifiedManager) fsManager() (cgroups.Manager, error) { + path, err := m.GetUnifiedPath() if err != nil { - return err + return nil, err } - err = freezer.Set(path, m.Cgroups) + return fs2.NewManager(m.Cgroups, path, false) +} + +func (m *UnifiedManager) Freeze(state configs.FreezerState) error { + fsMgr, err := m.fsManager() if err != nil { - m.Cgroups.Resources.Freezer = prevState return err } - return nil + return fsMgr.Freeze(state) } func (m *UnifiedManager) GetPids() ([]int, error) { @@ -305,44 +292,17 @@ func (m *UnifiedManager) GetAllPids() ([]int, error) { } func (m *UnifiedManager) GetStats() (*cgroups.Stats, error) { - m.mu.Lock() - defer m.mu.Unlock() - stats := cgroups.NewStats() - for name, path := range m.Paths { - sys, err := unifiedSubsystems.Get(name) - if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) { - continue - } - if err := sys.GetStats(path, stats); err != nil { - return nil, err - } + fsMgr, err := m.fsManager() + if err != nil { + return nil, err } - - return stats, nil + return fsMgr.GetStats() } func (m *UnifiedManager) Set(container *configs.Config) error { - // If Paths are set, then we are just joining cgroups paths - // and there is no need to set any values. - if m.Cgroups.Paths != nil { - return nil - } - for _, sys := range unifiedSubsystems { - // Get the subsystem path, but don't error out for not found cgroups. - path, err := getSubsystemPath(container.Cgroups, sys.Name()) - if err != nil && !cgroups.IsNotFound(err) { - return err - } - - if err := sys.Set(path, container.Cgroups); err != nil { - return err - } - } - - if m.Paths["cpu"] != "" { - if err := fs.CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil { - return err - } + fsMgr, err := m.fsManager() + if err != nil { + return err } - return nil + return fsMgr.Set(container) } diff --git a/libcontainer/factory_linux.go b/libcontainer/factory_linux.go index 78a8c0a8135..437633c6e49 100644 --- a/libcontainer/factory_linux.go +++ b/libcontainer/factory_linux.go @@ -14,12 +14,14 @@ import ( "github.com/cyphar/filepath-securejoin" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/cgroups/fs" + "github.com/opencontainers/runc/libcontainer/cgroups/fs2" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/configs/validate" "github.com/opencontainers/runc/libcontainer/intelrdt" "github.com/opencontainers/runc/libcontainer/mount" "github.com/opencontainers/runc/libcontainer/utils" + "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -59,10 +61,37 @@ func SystemdCgroups(l *LinuxFactory) error { return nil } +func getUnifiedPath(paths map[string]string) string { + unifiedPath := "" + for k, v := range paths { + if unifiedPath == "" { + unifiedPath = v + } else if v != unifiedPath { + panic(errors.Errorf("expected %q path to be unified path %q, got %q", k, unifiedPath, v)) + } + } + // can be empty + return unifiedPath +} + +func cgroupfs2(l *LinuxFactory, rootless bool) error { + l.NewCgroupsManager = func(config *configs.Cgroup, paths map[string]string) cgroups.Manager { + m, err := fs2.NewManager(config, getUnifiedPath(paths), rootless) + if err != nil { + panic(err) + } + return m + } + return nil +} + // Cgroupfs is an options func to configure a LinuxFactory to return containers // that use the native cgroups filesystem implementation to create and manage // cgroups. func Cgroupfs(l *LinuxFactory) error { + if cgroups.IsCgroup2UnifiedMode() { + return cgroupfs2(l, false) + } l.NewCgroupsManager = func(config *configs.Cgroup, paths map[string]string) cgroups.Manager { return &fs.Manager{ Cgroups: config, @@ -79,6 +108,9 @@ func Cgroupfs(l *LinuxFactory) error { // during rootless container (including euid=0 in userns) setup (while still allowing cgroup usage if // they've been set up properly). func RootlessCgroupfs(l *LinuxFactory) error { + if cgroups.IsCgroup2UnifiedMode() { + return cgroupfs2(l, true) + } l.NewCgroupsManager = func(config *configs.Cgroup, paths map[string]string) cgroups.Manager { return &fs.Manager{ Cgroups: config, From ec49f98d72aa7d3fc61eb0d9f4dd7915abcad114 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 6 Dec 2019 15:53:07 +0900 Subject: [PATCH 2/2] fs2: support legacy device spec (to pass CI) Signed-off-by: Akihiro Suda --- libcontainer/cgroups/fs2/devices.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libcontainer/cgroups/fs2/devices.go b/libcontainer/cgroups/fs2/devices.go index f852b53ede2..e0fd685402a 100644 --- a/libcontainer/cgroups/fs2/devices.go +++ b/libcontainer/cgroups/fs2/devices.go @@ -39,15 +39,23 @@ func canSkipEBPFError(cgroup *configs.Cgroup) bool { } func setDevices(dirPath string, cgroup *configs.Cgroup) error { - if cgroup.Resources.AllowAllDevices != nil { - // never set by OCI specconv - return errors.New("libcontainer AllowAllDevices is not supported, use Devices") + devices := cgroup.Devices + if allowAllDevices := cgroup.Resources.AllowAllDevices; allowAllDevices != nil { + // never set by OCI specconv, but *allowAllDevices=false is still used by the integration test + if *allowAllDevices == true { + return errors.New("libcontainer AllowAllDevices is not supported, use Devices") + } + for _, ad := range cgroup.Resources.AllowedDevices { + d := *ad + d.Allow = true + devices = append(devices, &d) + } } if len(cgroup.Resources.DeniedDevices) != 0 { // never set by OCI specconv return errors.New("libcontainer DeniedDevices is not supported, use Devices") } - insts, license, err := devicefilter.DeviceFilter(cgroup.Devices) + insts, license, err := devicefilter.DeviceFilter(devices) if err != nil { return err }