Make CGroups CPU period configurable

README.md

@@ -131,6 +131,7 @@ additional details on each available environment variable.
 | `ECS_HOST_DATA_DIR` | `/var/lib/ecs` | The source directory on the host from which ECS_DATADIR is mounted. We use this to determine the source mount path for container metadata files in the case the ECS Agent is running as a container. We do not use this value in Windows because the ECS Agent is not running as container in Windows. | `/var/lib/ecs` | `Not used` |
 | `ECS_ENABLE_TASK_CPU_MEM_LIMIT` | `true` | Whether to enable task-level cpu and memory limits | `true` | `false` |
 | `ECS_CGROUP_PATH` | `/sys/fs/cgroup` | The root cgroup path that is expected by the ECS agent. This is the path that accessible from the agent mount. | `/sys/fs/cgroup` | Not applicable |
+| `ECS_CGROUP_CPU_PERIOD` | `10ms` | CGroups CPU period for task level limits. This value should be between 1ms to 100ms | `100ms` | Not applicable |
 | `ECS_ENABLE_CPU_UNBOUNDED_WINDOWS_WORKAROUND` | `true` | When `true`, ECS will allow CPU unbounded(CPU=`0`) tasks to run along with CPU bounded tasks in Windows. | Not applicable | `false` |
 | `ECS_TASK_METADATA_RPS_LIMIT` | `100,150` | Comma separated integer values for steady state and burst throttle limits for task metadata endpoint | `40,60` | `40,60` |
 | `ECS_SHARED_VOLUME_MATCH_FULL_CONFIG` | `true` | When `true`, ECS Agent will compare name, driver options, and labels to make sure volumes are identical. When `false`, Agent will short circuit shared volume comparison if the names match. This is the default Docker behavior. If a volume is shared across instances, this should be set to `false`. | `false` | `false`|

agent/api/task/task.go

@@ -235,7 +235,7 @@ func (task *Task) PostUnmarshalTask(cfg *config.Config,
 	// hook into this
 	task.adjustForPlatform(cfg)
 	if task.MemoryCPULimitsEnabled {
-		err := task.initializeCgroupResourceSpec(cfg.CgroupPath, resourceFields)
+		err := task.initializeCgroupResourceSpec(cfg.CgroupPath, cfg.CGroupCPUPeriod, resourceFields)
 		if err != nil {
 			seelog.Errorf("Task [%s]: could not intialize resource: %v", task.Arn, err)
 			return apierrors.NewResourceInitError(task.Arn, err)

agent/api/task/task_linux.go

@@ -33,6 +33,7 @@ import (
 
 const (
 	defaultCPUPeriod = 100 * time.Millisecond // 100ms
+
 	// With a 100ms CPU period, we can express 0.01 vCPU to 10 vCPUs
 	maxTaskVCPULimit = 10
 	// Reference: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html
@@ -51,12 +52,12 @@ func (task *Task) adjustForPlatform(cfg *config.Config) {
 	task.MemoryCPULimitsEnabled = cfg.TaskCPUMemLimit.Enabled()
 }
 
-func (task *Task) initializeCgroupResourceSpec(cgroupPath string, resourceFields *taskresource.ResourceFields) error {
+func (task *Task) initializeCgroupResourceSpec(cgroupPath string, cGroupCPUPeriod time.Duration, resourceFields *taskresource.ResourceFields) error {
 	cgroupRoot, err := task.BuildCgroupRoot()
 	if err != nil {
 		return errors.Wrapf(err, "cgroup resource: unable to determine cgroup root for task")
 	}
-	resSpec, err := task.BuildLinuxResourceSpec()
+	resSpec, err := task.BuildLinuxResourceSpec(cGroupCPUPeriod)
 	if err != nil {
 		return errors.Wrapf(err, "cgroup resource: unable to build resource spec for task")
 	}
@@ -85,13 +86,13 @@ func (task *Task) BuildCgroupRoot() (string, error) {
 }
 
 // BuildLinuxResourceSpec returns a linuxResources object for the task cgroup
-func (task *Task) BuildLinuxResourceSpec() (specs.LinuxResources, error) {
+func (task *Task) BuildLinuxResourceSpec(cGroupCPUPeriod time.Duration) (specs.LinuxResources, error) {
 	linuxResourceSpec := specs.LinuxResources{}
 
 	// If task level CPU limits are requested, set CPU quota + CPU period
 	// Else set CPU shares
 	if task.CPU > 0 {
-		linuxCPUSpec, err := task.buildExplicitLinuxCPUSpec()
+		linuxCPUSpec, err := task.buildExplicitLinuxCPUSpec(cGroupCPUPeriod)
 		if err != nil {
 			return specs.LinuxResources{}, err
 		}
@@ -116,13 +117,13 @@ func (task *Task) BuildLinuxResourceSpec() (specs.LinuxResources, error) {
 
 // buildExplicitLinuxCPUSpec builds CPU spec when task CPU limits are
 // explicitly requested
-func (task *Task) buildExplicitLinuxCPUSpec() (specs.LinuxCPU, error) {
+func (task *Task) buildExplicitLinuxCPUSpec(cGroupCPUPeriod time.Duration) (specs.LinuxCPU, error) {
 	if task.CPU > maxTaskVCPULimit {
 		return specs.LinuxCPU{},
 			errors.Errorf("task CPU spec builder: unsupported CPU limits, requested=%f, max-supported=%d",
 				task.CPU, maxTaskVCPULimit)
 	}
-	taskCPUPeriod := uint64(defaultCPUPeriod / time.Microsecond)
+	taskCPUPeriod := uint64(cGroupCPUPeriod / time.Microsecond)
 	taskCPUQuota := int64(task.CPU * float64(taskCPUPeriod))
 
 	// TODO: DefaultCPUPeriod only permits 10VCPUs.

agent/api/task/task_linux_test.go

@@ -111,6 +111,7 @@ func TestBuildCgroupRootErrorPath(t *testing.T) {
 // TestBuildLinuxResourceSpecCPUMem validates the linux resource spec builder
 func TestBuildLinuxResourceSpecCPUMem(t *testing.T) {
 	taskMemoryLimit := int64(taskMemoryLimit)
+	defaultCPUPeriod := 100 * time.Millisecond
 
 	task := &Task{
 		Arn:    validTaskArn,
@@ -131,14 +132,16 @@ func TestBuildLinuxResourceSpecCPUMem(t *testing.T) {
 		},
 	}
 
-	linuxResourceSpec, err := task.BuildLinuxResourceSpec()
+	linuxResourceSpec, err := task.BuildLinuxResourceSpec(defaultCPUPeriod)
 
 	assert.NoError(t, err)
 	assert.EqualValues(t, expectedLinuxResourceSpec, linuxResourceSpec)
 }
 
 // TestBuildLinuxResourceSpecCPU validates the linux resource spec builder
 func TestBuildLinuxResourceSpecCPU(t *testing.T) {
+	defaultCPUPeriod := 100 * time.Millisecond
+
 	task := &Task{
 		Arn: validTaskArn,
 		CPU: float64(taskVCPULimit),
@@ -153,7 +156,7 @@ func TestBuildLinuxResourceSpecCPU(t *testing.T) {
 		},
 	}
 
-	linuxResourceSpec, err := task.BuildLinuxResourceSpec()
+	linuxResourceSpec, err := task.BuildLinuxResourceSpec(defaultCPUPeriod)
 
 	assert.NoError(t, err)
 	assert.EqualValues(t, expectedLinuxResourceSpec, linuxResourceSpec)
@@ -176,7 +179,7 @@ func TestBuildLinuxResourceSpecWithoutTaskCPULimits(t *testing.T) {
 		},
 	}
 
-	linuxResourceSpec, err := task.BuildLinuxResourceSpec()
+	linuxResourceSpec, err := task.BuildLinuxResourceSpec(100 * time.Millisecond)
 
 	assert.NoError(t, err)
 	assert.EqualValues(t, expectedLinuxResourceSpec, linuxResourceSpec)
@@ -200,7 +203,7 @@ func TestBuildLinuxResourceSpecWithoutTaskCPUWithContainerCPULimits(t *testing.T
 		},
 	}
 
-	linuxResourceSpec, err := task.BuildLinuxResourceSpec()
+	linuxResourceSpec, err := task.BuildLinuxResourceSpec(100 * time.Millisecond)
 
 	assert.NoError(t, err)
 	assert.EqualValues(t, expectedLinuxResourceSpec, linuxResourceSpec)
@@ -223,7 +226,7 @@ func TestBuildLinuxResourceSpecInvalidMem(t *testing.T) {
 	}
 
 	expectedLinuxResourceSpec := specs.LinuxResources{}
-	linuxResourceSpec, err := task.BuildLinuxResourceSpec()
+	linuxResourceSpec, err := task.BuildLinuxResourceSpec(100 * time.Millisecond)
 
 	assert.Error(t, err)
 	assert.EqualValues(t, expectedLinuxResourceSpec, linuxResourceSpec)
@@ -369,7 +372,7 @@ func TestInitCgroupResourceSpecHappyPath(t *testing.T) {
 	defer ctrl.Finish()
 	mockControl := mock_control.NewMockControl(ctrl)
 	mockIO := mock_ioutilwrapper.NewMockIOUtil(ctrl)
-	assert.NoError(t, task.initializeCgroupResourceSpec("cgroupPath", &taskresource.ResourceFields{
+	assert.NoError(t, task.initializeCgroupResourceSpec("cgroupPath", 100*time.Millisecond, &taskresource.ResourceFields{
 		Control: mockControl,
 		ResourceFieldsCommon: &taskresource.ResourceFieldsCommon{
 			IOUtil: mockIO,
@@ -393,7 +396,7 @@ func TestInitCgroupResourceSpecInvalidARN(t *testing.T) {
 		MemoryCPULimitsEnabled: true,
 		ResourcesMapUnsafe:     make(map[string][]taskresource.TaskResource),
 	}
-	assert.Error(t, task.initializeCgroupResourceSpec("", nil))
+	assert.Error(t, task.initializeCgroupResourceSpec("", time.Millisecond, nil))
 	assert.Equal(t, 0, len(task.GetResources()))
 	assert.Equal(t, 0, len(task.Containers[0].TransitionDependenciesMap))
 }
@@ -414,7 +417,7 @@ func TestInitCgroupResourceSpecInvalidMem(t *testing.T) {
 		MemoryCPULimitsEnabled: true,
 		ResourcesMapUnsafe:     make(map[string][]taskresource.TaskResource),
 	}
-	assert.Error(t, task.initializeCgroupResourceSpec("", nil))
+	assert.Error(t, task.initializeCgroupResourceSpec("", time.Millisecond, nil))
 	assert.Equal(t, 0, len(task.GetResources()))
 	assert.Equal(t, 0, len(task.Containers[0].TransitionDependenciesMap))
 }

agent/api/task/task_unsupported.go

@@ -26,6 +26,7 @@ import (
 
 const (
 	defaultCPUPeriod = 100 * time.Millisecond // 100ms
+
 	// With a 100ms CPU period, we can express 0.01 vCPU to 10 vCPUs
 	maxTaskVCPULimit = 10
 	// Reference: http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html
@@ -46,7 +47,7 @@ func (task *Task) adjustForPlatform(cfg *config.Config) {
 
 func getCanonicalPath(path string) string { return path }
 
-func (task *Task) initializeCgroupResourceSpec(cgroupPath string, resourceFields *taskresource.ResourceFields) error {
+func (task *Task) initializeCgroupResourceSpec(cgroupPath string, cGroupCPUPeriod time.Duration, resourceFields *taskresource.ResourceFields) error {
 	return nil
 }
 

agent/api/task/task_windows.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/aws/amazon-ecs-agent/agent/config"
 	"github.com/aws/amazon-ecs-agent/agent/taskresource"
@@ -120,6 +121,6 @@ func (task *Task) dockerCPUShares(containerCPU uint) int64 {
 	return int64(containerCPU)
 }
 
-func (task *Task) initializeCgroupResourceSpec(cgroupPath string, resourceFields *taskresource.ResourceFields) error {
+func (task *Task) initializeCgroupResourceSpec(cgroupPath string, cGroupCPUPeriod time.Duration, resourceFields *taskresource.ResourceFields) error {
 	return errors.New("unsupported platform")
 }

agent/config/config.go

@@ -130,6 +130,11 @@ const (
 	//Known cached image names
 	CachedImageNamePauseContainer = "amazon/amazon-ecs-pause:0.1.0"
 	CachedImageNameAgentContainer = "amazon/amazon-ecs-agent:latest"
+
+	// DefaultCPUPeriod is set to 100 ms to set CFS period and quota for task limits
+	defaultCGroupCPUPeriod = 100 * time.Millisecond
+	maximumCGroupCPUPeriod = 100 * time.Millisecond
+	minimumCGroupCPUPeriod = 1 * time.Millisecond
 )
 
 const (
@@ -532,6 +537,7 @@ func environmentConfig() (Config, error) {
 		ContainerInstancePropagateTagsFrom:  parseContainerInstancePropagateTagsFrom(),
 		PollMetrics:                         utils.ParseBool(os.Getenv("ECS_POLL_METRICS"), false),
 		PollingMetricsWaitDuration:          parseEnvVariableDuration("ECS_POLLING_METRICS_WAIT_DURATION"),
+		CGroupCPUPeriod:                     parseCGroupCPUPeriod(),
 	}, err
 }
 

agent/config/config_test.go

@@ -114,10 +114,12 @@ func TestEnvironmentConfig(t *testing.T) {
 	defer setTestEnv("ECS_SHARED_VOLUME_MATCH_FULL_CONFIG", "true")()
 	defer setTestEnv("ECS_POLL_METRICS", "true")()
 	defer setTestEnv("ECS_POLLING_METRICS_WAIT_DURATION", "10s")()
+	defer setTestEnv("ECS_CGROUP_CPU_PERIOD", "")
 	additionalLocalRoutesJSON := `["1.2.3.4/22","5.6.7.8/32"]`
 	setTestEnv("ECS_AWSVPC_ADDITIONAL_LOCAL_ROUTES", additionalLocalRoutesJSON)
 	setTestEnv("ECS_ENABLE_CONTAINER_METADATA", "true")
 	setTestEnv("ECS_HOST_DATA_DIR", "/etc/ecs/")
+	setTestEnv("ECS_CGROUP_CPU_PERIOD", "10ms")
 
 	conf, err := environmentConfig()
 	assert.NoError(t, err)
@@ -157,6 +159,7 @@ func TestEnvironmentConfig(t *testing.T) {
 	assert.Equal(t, 1000, conf.TaskMetadataSteadyStateRate)
 	assert.Equal(t, 1100, conf.TaskMetadataBurstRate)
 	assert.True(t, conf.SharedVolumeMatchFullConfig, "Wrong value for SharedVolumeMatchFullConfig")
+	assert.Equal(t, 10*time.Millisecond, conf.CGroupCPUPeriod)
 }
 
 func TestTrimWhitespaceWhenCreating(t *testing.T) {

agent/config/config_unix.go

@@ -79,6 +79,7 @@ func DefaultConfig() Config {
 		PrometheusMetricsEnabled:            false,
 		PollMetrics:                         false,
 		PollingMetricsWaitDuration:          DefaultPollingMetricsWaitDuration,
+		CGroupCPUPeriod:                     defaultCGroupCPUPeriod,
 	}
 }
 

agent/config/config_unix_test.go

@@ -65,6 +65,7 @@ func TestConfigDefault(t *testing.T) {
 	assert.Equal(t, DefaultTaskMetadataBurstRate, cfg.TaskMetadataBurstRate,
 		"Default TaskMetadataBurstRate is set incorrectly")
 	assert.False(t, cfg.SharedVolumeMatchFullConfig, "Default SharedVolumeMatchFullConfig set incorrectly")
+	assert.Equal(t, defaultCGroupCPUPeriod, cfg.CGroupCPUPeriod, "CFS cpu period set incorrectly")
 }
 
 // TestConfigFromFile tests the configuration can be read from file
@@ -201,3 +202,40 @@ func setupFileConfiguration(t *testing.T, configContent string) string {
 
 	return file.Name()
 }
+
+func TestOverrideDefaultCPUPeriod(t *testing.T) {
+	defer setTestRegion()()
+	defer os.Setenv("ECS_CGROUP_CPU_PERIOD", "100ms")
+
+	os.Setenv("ECS_CGROUP_CPU_PERIOD", "10ms")
+	conf, err := NewConfig(ec2.NewBlackholeEC2MetadataClient())
+	assert.NoError(t, err)
+	assert.Equal(t, 10*time.Millisecond, conf.CGroupCPUPeriod, "Wrong value for CGroupCPUPeriod")
+}
+
+func TestDefaultCPUPeriod(t *testing.T) {
+	defer setTestRegion()()
+	conf, err := NewConfig(ec2.NewBlackholeEC2MetadataClient())
+	assert.NoError(t, err)
+	assert.Equal(t, defaultCGroupCPUPeriod, conf.CGroupCPUPeriod, "Wrong value for CGroupCPUPeriod")
+}
+
+func TestCPUPeriodUpperBoundLimit(t *testing.T) {
+	defer setTestRegion()()
+	defer os.Setenv("ECS_CGROUP_CPU_PERIOD", "100ms")
+
+	os.Setenv("ECS_CGROUP_CPU_PERIOD", "110ms")
+	conf, err := NewConfig(ec2.NewBlackholeEC2MetadataClient())
+	assert.NoError(t, err)
+	assert.Equal(t, defaultCGroupCPUPeriod, conf.CGroupCPUPeriod, "Wrong value for CGroupCPUPeriod")
+}
+
+func TestCPUPeriodLowerBoundLimit(t *testing.T) {
+	defer setTestRegion()()
+	defer os.Setenv("ECS_CGROUP_CPU_PERIOD", "100ms")
+
+	os.Setenv("ECS_CGROUP_CPU_PERIOD", "100us")
+	conf, err := NewConfig(ec2.NewBlackholeEC2MetadataClient())
+	assert.NoError(t, err)
+	assert.Equal(t, defaultCGroupCPUPeriod, conf.CGroupCPUPeriod, "Wrong value for CGroupCPUPeriod")
+}

agent/config/parse.go

@@ -307,3 +307,23 @@ func parseImageCleanupExclusionList(envVar string) []string {
 	}
 	return imageCleanupExclusionList
 }
+
+func parseCGroupCPUPeriod() time.Duration {
+	envVal := os.Getenv("ECS_CGROUP_CPU_PERIOD")
+	if envVal == "" {
+		seelog.Debugf("Environment variable empty: ECS_CGROUP_CPU_PERIOD")
+	} else {
+		var err error
+		duration, err := time.ParseDuration(envVal)
+		if err != nil {
+			seelog.Warnf("Could not parse duration value: %v for Environment Variable ECS_CGROUP_CPU_PERIOD is not correct: %v, using default value instead", envVal, err)
+		} else {
+			if duration >= minimumCGroupCPUPeriod && duration <= maximumCGroupCPUPeriod {
+				return duration
+			} else {
+				seelog.Warnf("CPU Period duration value: %v for Environment Variable ECS_CGROUP_CPU_PERIOD is not within [%v, %v], using default value instead", envVal, minimumCGroupCPUPeriod, maximumCGroupCPUPeriod)
+			}
+		}
+	}
+	return defaultCGroupCPUPeriod
+}

agent/config/types.go

@@ -271,4 +271,7 @@ type Config struct {
 
 	// ImageCleanupExclusionList is the list of image names customers want to keep for their own use and delete automatically
 	ImageCleanupExclusionList []string
+
+	// CGroupCPUPeriod is config option to set different CFS quota and period values in microsecond, defaults to 100 ms
+	CGroupCPUPeriod time.Duration
 }