Merge pull request #1067 from allencloud/add-cpu-options

feature: add cpu period and quota for containers

apis/swagger.yml

@@ -2083,12 +2083,15 @@ definitions:
           The length of a CPU period in microseconds.
         type: "integer"
         format: "int64"
+        minimum: 1000
+        maximum: 1000000
       CpuQuota:
         description: |
           CPU CFS (Completely Fair Scheduler) quota.
           Microseconds of CPU time that the container can get in a CPU period."
         type: "integer"
         format: "int64"
+        minimum: 1000
       CpuRealtimePeriod:
         description: "The length of a CPU real-time period in microseconds. Set to 0 to allocate no time allocated to real-time tasks."
         type: "integer"

cli/common_flags.go

@@ -20,6 +20,8 @@ func addCommonFlags(flagSet *pflag.FlagSet) *container {
 	flagSet.Int64Var(&c.cpushare, "cpu-share", 0, "CPU shares (relative weight)")
 	flagSet.StringVar(&c.cpusetcpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)")
 	flagSet.StringVar(&c.cpusetmems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)")
+	flagSet.Int64Var(&c.cpuperiod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period, range is in [1000(1ms),1000000(1s)]")
+	flagSet.Int64Var(&c.cpuquota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota, range is in [1000,∞)")
 
 	// device related options
 	flagSet.StringSliceVarP(&c.devices, "device", "", nil, "Add a host device to the container")

cli/container.go

@@ -10,20 +10,31 @@ import (
 )
 
 type container struct {
-	labels           []string
-	name             string
-	tty              bool
-	volume           []string
-	runtime          string
-	env              []string
-	entrypoint       string
-	workdir          string
-	user             string
-	groupAdd         []string
-	hostname         string
-	cpushare         int64
-	cpusetcpus       string
-	cpusetmems       string
+	labels     []string
+	name       string
+	tty        bool
+	volume     []string
+	runtime    string
+	env        []string
+	entrypoint string
+	workdir    string
+	user       string
+	groupAdd   []string
+	hostname   string
+
+	blkioWeight          uint16
+	blkioWeightDevice    WeightDevice
+	blkioDeviceReadBps   ThrottleBpsDevice
+	blkioDeviceWriteBps  ThrottleBpsDevice
+	blkioDeviceReadIOps  ThrottleIOpsDevice
+	blkioDeviceWriteIOps ThrottleIOpsDevice
+
+	cpushare   int64
+	cpusetcpus string
+	cpusetmems string
+	cpuperiod  int64
+	cpuquota   int64
+
 	memory           string
 	memorySwap       string
 	memorySwappiness int64
@@ -34,33 +45,26 @@ type container struct {
 	scheLatSwitch       int64
 	oomKillDisable      bool
 
-	devices              []string
-	enableLxcfs          bool
-	privileged           bool
-	restartPolicy        string
-	ipcMode              string
-	pidMode              string
-	utsMode              string
-	sysctls              []string
-	networks             []string
-	ports                []string
-	expose               []string
-	publicAll            bool
-	securityOpt          []string
-	capAdd               []string
-	capDrop              []string
-	blkioWeight          uint16
-	blkioWeightDevice    WeightDevice
-	blkioDeviceReadBps   ThrottleBpsDevice
-	blkioDeviceWriteBps  ThrottleBpsDevice
-	blkioDeviceReadIOps  ThrottleIOpsDevice
-	blkioDeviceWriteIOps ThrottleIOpsDevice
-	IntelRdtL3Cbm        string
-	diskQuota            []string
-	oomScoreAdj          int64
-	specAnnotation       []string
-
-	cgroupParent string
+	devices        []string
+	enableLxcfs    bool
+	privileged     bool
+	restartPolicy  string
+	ipcMode        string
+	pidMode        string
+	utsMode        string
+	sysctls        []string
+	networks       []string
+	ports          []string
+	expose         []string
+	publicAll      bool
+	securityOpt    []string
+	capAdd         []string
+	capDrop        []string
+	IntelRdtL3Cbm  string
+	diskQuota      []string
+	oomScoreAdj    int64
+	specAnnotation []string
+	cgroupParent   string
 
 	//add for rich container mode
 	rich       bool
@@ -130,6 +134,14 @@ func (c *container) config() (*types.ContainerCreateConfig, error) {
 		return nil, err
 	}
 
+	if err := opts.ValidateCPUPeriod(c.cpuperiod); err != nil {
+		return nil, err
+	}
+
+	if err := opts.ValidateCPUQuota(c.cpuquota); err != nil {
+		return nil, err
+	}
+
 	networkingConfig, networkMode, err := opts.ParseNetworks(c.networks)
 	if err != nil {
 		return nil, err
@@ -175,10 +187,14 @@ func (c *container) config() (*types.ContainerCreateConfig, error) {
 			Binds:   c.volume,
 			Runtime: c.runtime,
 			Resources: types.Resources{
-				CPUShares:        c.cpushare,
-				CpusetCpus:       c.cpusetcpus,
-				CpusetMems:       c.cpusetmems,
-				Devices:          deviceMappings,
+				// cpu
+				CPUShares:  c.cpushare,
+				CpusetCpus: c.cpusetcpus,
+				CpusetMems: c.cpusetmems,
+				CPUPeriod:  c.cpuperiod,
+				CPUQuota:   c.cpuquota,
+
+				// memory
 				Memory:           memory,
 				MemorySwap:       memorySwap,
 				MemorySwappiness: &c.memorySwappiness,
@@ -196,9 +212,10 @@ func (c *container) config() (*types.ContainerCreateConfig, error) {
 				BlkioDeviceReadIOps:  c.blkioDeviceReadIOps.value(),
 				BlkioDeviceWriteBps:  c.blkioDeviceWriteBps.value(),
 				BlkioDeviceWriteIOps: c.blkioDeviceWriteIOps.value(),
-				IntelRdtL3Cbm:        intelRdtL3Cbm,
 
-				CgroupParent: c.cgroupParent,
+				Devices:       deviceMappings,
+				IntelRdtL3Cbm: intelRdtL3Cbm,
+				CgroupParent:  c.cgroupParent,
 			},
 			EnableLxcfs:   c.enableLxcfs,
 			Privileged:    c.privileged,

daemon/mgr/spec.go

@@ -34,6 +34,8 @@ var setupFunc = []SetupFunc{
 	// cgroup
 	setupCgroupCPUShare,
 	setupCgroupCPUSet,
+	setupCgroupCPUPeriod,
+	setupCgroupCPUQuota,
 	setupCgroupMemory,
 	setupCgroupMemorySwap,
 	setupCgroupMemorySwappiness,

daemon/mgr/spec_cgroup_cpu.go

@@ -28,3 +28,25 @@ func setupCgroupCPUSet(ctx context.Context, meta *ContainerMeta, spec *SpecWrapp
 	cpu.Mems = meta.HostConfig.CpusetMems
 	return nil
 }
+
+func setupCgroupCPUPeriod(ctx context.Context, meta *ContainerMeta, spec *SpecWrapper) error {
+	s := spec.s
+	if s.Linux.Resources.CPU == nil {
+		s.Linux.Resources.CPU = &specs.LinuxCPU{}
+	}
+	cpu := s.Linux.Resources.CPU
+	period := uint64(meta.HostConfig.CPUPeriod)
+	cpu.Period = &period
+	return nil
+}
+
+func setupCgroupCPUQuota(ctx context.Context, meta *ContainerMeta, spec *SpecWrapper) error {
+	s := spec.s
+	if s.Linux.Resources.CPU == nil {
+		s.Linux.Resources.CPU = &specs.LinuxCPU{}
+	}
+	cpu := s.Linux.Resources.CPU
+	quota := meta.HostConfig.CPUQuota
+	cpu.Quota = &quota
+	return nil
+}

pkg/opts/cpu.go

@@ -0,0 +1,27 @@
+package opts
+
+import (
+	"fmt"
+)
+
+// ValidateCPUPeriod validates CPU options for container.
+func ValidateCPUPeriod(period int64) error {
+	if period == 0 {
+		return nil
+	}
+	if period < 1000 || period > 1000000 {
+		return fmt.Errorf("CPU cfs period  %d cannot be less than 1ms (i.e. 1000) or larger than 1s (i.e. 1000000)", period)
+	}
+	return nil
+}
+
+// ValidateCPUQuota validates CPU options for container.
+func ValidateCPUQuota(quota int64) error {
+	if quota == 0 {
+		return nil
+	}
+	if quota < 1000 {
+		return fmt.Errorf("CPU cfs quota %d cannot be less than 1ms (i.e. 1000)", quota)
+	}
+	return nil
+}

test/cli_run_test.go

@@ -463,8 +463,15 @@ func (suite *PouchRunSuite) TestRunWithMemoryswappiness(c *check.C) {
 // TestRunWithCPULimit tests CPU related flags.
 func (suite *PouchRunSuite) TestRunWithCPULimit(c *check.C) {
 	cname := "TestRunWithCPULimit"
-	command.PouchRun("run", "-d", "--cpuset-cpus", "0", "--cpuset-mems", "0",
-		"--cpu-share", "1000", "--name", cname, busyboxImage, "sleep", "10000").Stdout()
+	command.PouchRun("run", "-d",
+		"--cpuset-cpus", "0",
+		"--cpuset-mems", "0",
+		"--cpu-share", "1000",
+		"--cpu-period", "1000",
+		"--cpu-quota", "1000",
+		"--name", cname,
+		busyboxImage,
+		"sleep", "10000").Stdout()
 
 	// test if the value is in inspect result
 	output := command.PouchRun("inspect", cname).Stdout()
@@ -473,9 +480,12 @@ func (suite *PouchRunSuite) TestRunWithCPULimit(c *check.C) {
 		c.Errorf("failed to decode inspect output: %v", err)
 	}
 
+	// check whether the user setting options are in containers' metadata
 	c.Assert(result.HostConfig.CpusetMems, check.Equals, "0")
 	c.Assert(result.HostConfig.CPUShares, check.Equals, int64(1000))
 	c.Assert(result.HostConfig.CpusetCpus, check.Equals, "0")
+	c.Assert(result.HostConfig.CPUPeriod, check.Equals, int64(1000))
+	c.Assert(result.HostConfig.CPUQuota, check.Equals, int64(1000))
 
 	// test if cgroup has record the real value
 	containerID := result.ID
@@ -491,6 +501,14 @@ func (suite *PouchRunSuite) TestRunWithCPULimit(c *check.C) {
 		path := fmt.Sprintf("/sys/fs/cgroup/cpu/default/%s/cpu.shares", containerID)
 		checkFileContains(c, path, "1000")
 	}
+	{
+		path := fmt.Sprintf("/sys/fs/cgroup/cpu/default/%s/cpu.cfs_period_us", containerID)
+		checkFileContains(c, path, "1000")
+	}
+	{
+		path := fmt.Sprintf("/sys/fs/cgroup/cpu/default/%s/cpu.cfs_quota_us", containerID)
+		checkFileContains(c, path, "1000")
+	}
 
 	DelContainerForceMultyTime(c, cname)
 }
@@ -705,7 +723,7 @@ func (suite *PouchRunSuite) TestRunWithHostFileVolume(c *check.C) {
 	filepath := "/tmp/TestRunWithHostFileVolume.md"
 	icmd.RunCommand("touch", filepath).Assert(c, icmd.Success)
 
-	cname := "TestRunWithCPULimit"
+	cname := "TestRunWithHostFileVolume"
 	command.PouchRun("run", "-d", "--name", cname, "-v", fmt.Sprintf("%s:%s", filepath, filepath), busyboxImage).Assert(c, icmd.Success)
 
 	DelContainerForceMultyTime(c, cname)

test/z_cli_daemon_test.go

@@ -177,7 +177,7 @@ func (suite *PouchDaemonSuite) TestDaemonRestart(c *check.C) {
 	output := command.PouchRun("inspect", "--host", daemon.Listen, cname).Stdout()
 	result := &types.ContainerJSON{}
 	if err := json.Unmarshal([]byte(output), result); err != nil {
-		c.Fatal("failed to decode inspect output: %v", err)
+		c.Fatalf("failed to decode inspect output: %v", err)
 	}
 	c.Assert(string(result.State.Status), check.Equals, "running")
 }