From 5940044177882ef3d48054260d574bf9c54a29b3 Mon Sep 17 00:00:00 2001 From: Allen Sun Date: Sun, 8 Apr 2018 15:33:25 +0800 Subject: [PATCH] feature: add cpu period and quota for containers Signed-off-by: Allen Sun --- apis/swagger.yml | 3 + apis/types/resources.go | 43 +++++++++++++ cli/common_flags.go | 2 + cli/container.go | 111 ++++++++++++++++++++-------------- daemon/mgr/spec.go | 2 + daemon/mgr/spec_cgroup_cpu.go | 22 +++++++ pkg/opts/cpu.go | 27 +++++++++ test/cli_run_test.go | 24 +++++++- test/z_cli_daemon_test.go | 2 +- 9 files changed, 185 insertions(+), 51 deletions(-) create mode 100644 pkg/opts/cpu.go diff --git a/apis/swagger.yml b/apis/swagger.yml index 3d0480785..7d0f2a2ca 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -2083,12 +2083,15 @@ definitions: The length of a CPU period in microseconds. type: "integer" format: "int64" + minimum: 1000 + maximum: 1000000 CpuQuota: description: | CPU CFS (Completely Fair Scheduler) quota. Microseconds of CPU time that the container can get in a CPU period." type: "integer" format: "int64" + minimum: 1000 CpuRealtimePeriod: description: "The length of a CPU real-time period in microseconds. Set to 0 to allocate no time allocated to real-time tasks." type: "integer" diff --git a/apis/types/resources.go b/apis/types/resources.go index f09642a1b..41b794d48 100644 --- a/apis/types/resources.go +++ b/apis/types/resources.go @@ -61,11 +61,14 @@ type Resources struct { // CPU CFS (Completely Fair Scheduler) period. // The length of a CPU period in microseconds. // + // Maximum: 1e+06 + // Minimum: 1000 CPUPeriod int64 `json:"CpuPeriod,omitempty"` // CPU CFS (Completely Fair Scheduler) quota. // Microseconds of CPU time that the container can get in a CPU period." // + // Minimum: 1000 CPUQuota int64 `json:"CpuQuota,omitempty"` // The length of a CPU real-time period in microseconds. Set to 0 to allocate no time allocated to real-time tasks. @@ -261,6 +264,16 @@ func (m *Resources) Validate(formats strfmt.Registry) error { res = append(res, err) } + if err := m.validateCPUPeriod(formats); err != nil { + // prop + res = append(res, err) + } + + if err := m.validateCPUQuota(formats); err != nil { + // prop + res = append(res, err) + } + if err := m.validateDeviceCgroupRules(formats); err != nil { // prop res = append(res, err) @@ -459,6 +472,36 @@ func (m *Resources) validateBlkioWeightDevice(formats strfmt.Registry) error { return nil } +func (m *Resources) validateCPUPeriod(formats strfmt.Registry) error { + + if swag.IsZero(m.CPUPeriod) { // not required + return nil + } + + if err := validate.MinimumInt("CpuPeriod", "body", int64(m.CPUPeriod), 1000, false); err != nil { + return err + } + + if err := validate.MaximumInt("CpuPeriod", "body", int64(m.CPUPeriod), 1e+06, false); err != nil { + return err + } + + return nil +} + +func (m *Resources) validateCPUQuota(formats strfmt.Registry) error { + + if swag.IsZero(m.CPUQuota) { // not required + return nil + } + + if err := validate.MinimumInt("CpuQuota", "body", int64(m.CPUQuota), 1000, false); err != nil { + return err + } + + return nil +} + func (m *Resources) validateDeviceCgroupRules(formats strfmt.Registry) error { if swag.IsZero(m.DeviceCgroupRules) { // not required diff --git a/cli/common_flags.go b/cli/common_flags.go index 8274b81a2..7bc5f30de 100644 --- a/cli/common_flags.go +++ b/cli/common_flags.go @@ -20,6 +20,8 @@ func addCommonFlags(flagSet *pflag.FlagSet) *container { flagSet.Int64Var(&c.cpushare, "cpu-share", 0, "CPU shares (relative weight)") flagSet.StringVar(&c.cpusetcpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)") flagSet.StringVar(&c.cpusetmems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)") + flagSet.Int64Var(&c.cpuperiod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period, range is in [1000(1ms),1000000(1s)]") + flagSet.Int64Var(&c.cpuquota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota, range is in [1000,∞)") // device related options flagSet.StringSliceVarP(&c.devices, "device", "", nil, "Add a host device to the container") diff --git a/cli/container.go b/cli/container.go index 1b9c85f19..b09c9358c 100644 --- a/cli/container.go +++ b/cli/container.go @@ -10,20 +10,31 @@ import ( ) type container struct { - labels []string - name string - tty bool - volume []string - runtime string - env []string - entrypoint string - workdir string - user string - groupAdd []string - hostname string - cpushare int64 - cpusetcpus string - cpusetmems string + labels []string + name string + tty bool + volume []string + runtime string + env []string + entrypoint string + workdir string + user string + groupAdd []string + hostname string + + blkioWeight uint16 + blkioWeightDevice WeightDevice + blkioDeviceReadBps ThrottleBpsDevice + blkioDeviceWriteBps ThrottleBpsDevice + blkioDeviceReadIOps ThrottleIOpsDevice + blkioDeviceWriteIOps ThrottleIOpsDevice + + cpushare int64 + cpusetcpus string + cpusetmems string + cpuperiod int64 + cpuquota int64 + memory string memorySwap string memorySwappiness int64 @@ -34,33 +45,26 @@ type container struct { scheLatSwitch int64 oomKillDisable bool - devices []string - enableLxcfs bool - privileged bool - restartPolicy string - ipcMode string - pidMode string - utsMode string - sysctls []string - networks []string - ports []string - expose []string - publicAll bool - securityOpt []string - capAdd []string - capDrop []string - blkioWeight uint16 - blkioWeightDevice WeightDevice - blkioDeviceReadBps ThrottleBpsDevice - blkioDeviceWriteBps ThrottleBpsDevice - blkioDeviceReadIOps ThrottleIOpsDevice - blkioDeviceWriteIOps ThrottleIOpsDevice - IntelRdtL3Cbm string - diskQuota []string - oomScoreAdj int64 - specAnnotation []string - - cgroupParent string + devices []string + enableLxcfs bool + privileged bool + restartPolicy string + ipcMode string + pidMode string + utsMode string + sysctls []string + networks []string + ports []string + expose []string + publicAll bool + securityOpt []string + capAdd []string + capDrop []string + IntelRdtL3Cbm string + diskQuota []string + oomScoreAdj int64 + specAnnotation []string + cgroupParent string //add for rich container mode rich bool @@ -130,6 +134,14 @@ func (c *container) config() (*types.ContainerCreateConfig, error) { return nil, err } + if err := opts.ValidateCPUPeriod(c.cpuperiod); err != nil { + return nil, err + } + + if err := opts.ValidateCPUQuota(c.cpuquota); err != nil { + return nil, err + } + networkingConfig, networkMode, err := opts.ParseNetworks(c.networks) if err != nil { return nil, err @@ -175,10 +187,14 @@ func (c *container) config() (*types.ContainerCreateConfig, error) { Binds: c.volume, Runtime: c.runtime, Resources: types.Resources{ - CPUShares: c.cpushare, - CpusetCpus: c.cpusetcpus, - CpusetMems: c.cpusetmems, - Devices: deviceMappings, + // cpu + CPUShares: c.cpushare, + CpusetCpus: c.cpusetcpus, + CpusetMems: c.cpusetmems, + CPUPeriod: c.cpuperiod, + CPUQuota: c.cpuquota, + + // memory Memory: memory, MemorySwap: memorySwap, MemorySwappiness: &c.memorySwappiness, @@ -196,9 +212,10 @@ func (c *container) config() (*types.ContainerCreateConfig, error) { BlkioDeviceReadIOps: c.blkioDeviceReadIOps.value(), BlkioDeviceWriteBps: c.blkioDeviceWriteBps.value(), BlkioDeviceWriteIOps: c.blkioDeviceWriteIOps.value(), - IntelRdtL3Cbm: intelRdtL3Cbm, - CgroupParent: c.cgroupParent, + Devices: deviceMappings, + IntelRdtL3Cbm: intelRdtL3Cbm, + CgroupParent: c.cgroupParent, }, EnableLxcfs: c.enableLxcfs, Privileged: c.privileged, diff --git a/daemon/mgr/spec.go b/daemon/mgr/spec.go index 33e0e6cec..36ae4be37 100644 --- a/daemon/mgr/spec.go +++ b/daemon/mgr/spec.go @@ -34,6 +34,8 @@ var setupFunc = []SetupFunc{ // cgroup setupCgroupCPUShare, setupCgroupCPUSet, + setupCgroupCPUPeriod, + setupCgroupCPUQuota, setupCgroupMemory, setupCgroupMemorySwap, setupCgroupMemorySwappiness, diff --git a/daemon/mgr/spec_cgroup_cpu.go b/daemon/mgr/spec_cgroup_cpu.go index 6ffa03911..6a6d4711d 100644 --- a/daemon/mgr/spec_cgroup_cpu.go +++ b/daemon/mgr/spec_cgroup_cpu.go @@ -28,3 +28,25 @@ func setupCgroupCPUSet(ctx context.Context, meta *ContainerMeta, spec *SpecWrapp cpu.Mems = meta.HostConfig.CpusetMems return nil } + +func setupCgroupCPUPeriod(ctx context.Context, meta *ContainerMeta, spec *SpecWrapper) error { + s := spec.s + if s.Linux.Resources.CPU == nil { + s.Linux.Resources.CPU = &specs.LinuxCPU{} + } + cpu := s.Linux.Resources.CPU + period := uint64(meta.HostConfig.CPUPeriod) + cpu.Period = &period + return nil +} + +func setupCgroupCPUQuota(ctx context.Context, meta *ContainerMeta, spec *SpecWrapper) error { + s := spec.s + if s.Linux.Resources.CPU == nil { + s.Linux.Resources.CPU = &specs.LinuxCPU{} + } + cpu := s.Linux.Resources.CPU + quota := meta.HostConfig.CPUQuota + cpu.Quota = "a + return nil +} diff --git a/pkg/opts/cpu.go b/pkg/opts/cpu.go new file mode 100644 index 000000000..ea427ffb6 --- /dev/null +++ b/pkg/opts/cpu.go @@ -0,0 +1,27 @@ +package opts + +import ( + "fmt" +) + +// ValidateCPUPeriod validates CPU options for container. +func ValidateCPUPeriod(period int64) error { + if period == 0 { + return nil + } + if period < 1000 || period > 1000000 { + return fmt.Errorf("CPU cfs period %d cannot be less than 1ms (i.e. 1000) or larger than 1s (i.e. 1000000)", period) + } + return nil +} + +// ValidateCPUQuota validates CPU options for container. +func ValidateCPUQuota(quota int64) error { + if quota == 0 { + return nil + } + if quota < 1000 { + return fmt.Errorf("CPU cfs quota %d cannot be less than 1ms (i.e. 1000)", quota) + } + return nil +} diff --git a/test/cli_run_test.go b/test/cli_run_test.go index 6b24965de..5ddd744dc 100644 --- a/test/cli_run_test.go +++ b/test/cli_run_test.go @@ -463,8 +463,15 @@ func (suite *PouchRunSuite) TestRunWithMemoryswappiness(c *check.C) { // TestRunWithCPULimit tests CPU related flags. func (suite *PouchRunSuite) TestRunWithCPULimit(c *check.C) { cname := "TestRunWithCPULimit" - command.PouchRun("run", "-d", "--cpuset-cpus", "0", "--cpuset-mems", "0", - "--cpu-share", "1000", "--name", cname, busyboxImage, "sleep", "10000").Stdout() + command.PouchRun("run", "-d", + "--cpuset-cpus", "0", + "--cpuset-mems", "0", + "--cpu-share", "1000", + "--cpu-period", "1000", + "--cpu-quota", "1000", + "--name", cname, + busyboxImage, + "sleep", "10000").Stdout() // test if the value is in inspect result output := command.PouchRun("inspect", cname).Stdout() @@ -473,9 +480,12 @@ func (suite *PouchRunSuite) TestRunWithCPULimit(c *check.C) { c.Errorf("failed to decode inspect output: %v", err) } + // check whether the user setting options are in containers' metadata c.Assert(result.HostConfig.CpusetMems, check.Equals, "0") c.Assert(result.HostConfig.CPUShares, check.Equals, int64(1000)) c.Assert(result.HostConfig.CpusetCpus, check.Equals, "0") + c.Assert(result.HostConfig.CPUPeriod, check.Equals, int64(1000)) + c.Assert(result.HostConfig.CPUQuota, check.Equals, int64(1000)) // test if cgroup has record the real value containerID := result.ID @@ -491,6 +501,14 @@ func (suite *PouchRunSuite) TestRunWithCPULimit(c *check.C) { path := fmt.Sprintf("/sys/fs/cgroup/cpu/default/%s/cpu.shares", containerID) checkFileContains(c, path, "1000") } + { + path := fmt.Sprintf("/sys/fs/cgroup/cpu/default/%s/cpu.cfs_period_us", containerID) + checkFileContains(c, path, "1000") + } + { + path := fmt.Sprintf("/sys/fs/cgroup/cpu/default/%s/cpu.cfs_quota_us", containerID) + checkFileContains(c, path, "1000") + } DelContainerForceMultyTime(c, cname) } @@ -705,7 +723,7 @@ func (suite *PouchRunSuite) TestRunWithHostFileVolume(c *check.C) { filepath := "/tmp/TestRunWithHostFileVolume.md" icmd.RunCommand("touch", filepath).Assert(c, icmd.Success) - cname := "TestRunWithCPULimit" + cname := "TestRunWithHostFileVolume" command.PouchRun("run", "-d", "--name", cname, "-v", fmt.Sprintf("%s:%s", filepath, filepath), busyboxImage).Assert(c, icmd.Success) DelContainerForceMultyTime(c, cname) diff --git a/test/z_cli_daemon_test.go b/test/z_cli_daemon_test.go index fcaaa9c05..0b4ee280b 100644 --- a/test/z_cli_daemon_test.go +++ b/test/z_cli_daemon_test.go @@ -177,7 +177,7 @@ func (suite *PouchDaemonSuite) TestDaemonRestart(c *check.C) { output := command.PouchRun("inspect", "--host", daemon.Listen, cname).Stdout() result := &types.ContainerJSON{} if err := json.Unmarshal([]byte(output), result); err != nil { - c.Fatal("failed to decode inspect output: %v", err) + c.Fatalf("failed to decode inspect output: %v", err) } c.Assert(string(result.State.Status), check.Equals, "running") }