Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PLAT-713] Allow the CFS period to be tuned for containers #1

Closed
wants to merge 1 commit into from
Closed

[PLAT-713] Allow the CFS period to be tuned for containers #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions pkg/kubelet/cm/helpers_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,13 @@ const (
MilliCPUToCPU = 1000

// 100000 is equivalent to 100ms
QuotaPeriod = 100000
MinQuotaPeriod = 1000
DefaultQuotaPeriod int64 = 100000
MinQuotaPeriod int64 = 1000
)

// MilliCPUToQuota converts milliCPU to CFS quota and period values.
func MilliCPUToQuota(milliCPU int64) (quota int64, period int64) {
// MilliCPUToQuota takes milliCPU (along with a CFS period, in usec) and returns
// a CFS quota value
func MilliCPUToQuota(milliCPU, period int64) (quota int64) {
// CFS quota is measured in two values:
// - cfs_period_us=100ms (the amount of time to measure usage across)
// - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
Expand All @@ -53,11 +54,8 @@ func MilliCPUToQuota(milliCPU int64) (quota int64, period int64) {
return
}

// we set the period to 100ms by default
period = QuotaPeriod

// we then convert your milliCPU to a value normalized over a period
quota = (milliCPU * QuotaPeriod) / MilliCPUToCPU
quota = (milliCPU * period) / MilliCPUToCPU

// quota needs to be a minimum of 1ms.
if quota < MinQuotaPeriod {
Expand Down Expand Up @@ -90,20 +88,24 @@ func ResourceConfigForPod(pod *v1.Pod) *ResourceConfig {

cpuRequests := int64(0)
cpuLimits := int64(0)
cpuPeriod := DefaultQuotaPeriod
memoryLimits := int64(0)
if request, found := reqs[v1.ResourceCPU]; found {
cpuRequests = request.MilliValue()
}
if limit, found := limits[v1.ResourceCPU]; found {
cpuLimits = limit.MilliValue()
}
if limit, found := limits[v1.ResourceCPUPeriodUsec]; found {
cpuPeriod = limit.Value()
}
if limit, found := limits[v1.ResourceMemory]; found {
memoryLimits = limit.Value()
}

// convert to CFS values
cpuShares := MilliCPUToShares(cpuRequests)
cpuQuota, cpuPeriod := MilliCPUToQuota(cpuLimits)
cpuQuota := MilliCPUToQuota(cpuLimits, cpuPeriod)

// track if limits were applied for each resource.
memoryLimitsDeclared := true
Expand Down
137 changes: 104 additions & 33 deletions pkg/kubelet/cm/helpers_linux_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,14 @@ func TestResourceConfigForPod(t *testing.T) {
memoryQuantity := resource.MustParse("200Mi")
burstableMemory := memoryQuantity.Value()
burstablePartialShares := MilliCPUToShares(200)
burstableQuota, burstablePeriod := MilliCPUToQuota(200)
burstablePeriod := DefaultQuotaPeriod
burstableQuota := MilliCPUToQuota(200, burstablePeriod)
guaranteedShares := MilliCPUToShares(100)
guaranteedQuota, guaranteedPeriod := MilliCPUToQuota(100)
guaranteedPeriod := int64(10000)
guaranteedQuota := MilliCPUToQuota(100, guaranteedPeriod)
memoryQuantity = resource.MustParse("100Mi")
guaranteedMemory := memoryQuantity.Value()

testCases := map[string]struct {
pod *v1.Pod
expected *ResourceConfig
Expand All @@ -67,19 +70,30 @@ func TestResourceConfigForPod(t *testing.T) {
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: getResourceRequirements(getResourceList("", ""), getResourceList("", "")),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{},
Limits: v1.ResourceList{},
},
},
},
},
},
expected: &ResourceConfig{CpuShares: &minShares},
expected: &ResourceConfig{
CpuShares: &minShares,
},
},
"burstable-no-limits": {
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
},
Limits: v1.ResourceList{},
},
},
},
},
Expand All @@ -91,39 +105,86 @@ func TestResourceConfigForPod(t *testing.T) {
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("200m"),
v1.ResourceMemory: resource.MustParse("200Mi"),
},
},
},
},
},
},
expected: &ResourceConfig{CpuShares: &burstableShares, CpuQuota: &burstableQuota, CpuPeriod: &burstablePeriod, Memory: &burstableMemory},
expected: &ResourceConfig{
CpuShares: &burstableShares,
CpuQuota: &burstableQuota,
CpuPeriod: &burstablePeriod,
Memory: &burstableMemory,
},
},
"burstable-partial-limits": {
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi")),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("200m"),
v1.ResourceMemory: resource.MustParse("200Mi"),
},
},
},
{
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", "")),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
},
Limits: v1.ResourceList{},
},
},
},
},
},
expected: &ResourceConfig{CpuShares: &burstablePartialShares},
expected: &ResourceConfig{
CpuShares: &burstablePartialShares,
},
},
"guaranteed": {
pod: &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("100m", "100Mi")),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
v1.ResourceCPUPeriodUsec: resource.MustParse("10000"),
},
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("100m"),
v1.ResourceMemory: resource.MustParse("100Mi"),
v1.ResourceCPUPeriodUsec: resource.MustParse("10000"),
},
},
},
},
},
},
expected: &ResourceConfig{CpuShares: &guaranteedShares, CpuQuota: &guaranteedQuota, CpuPeriod: &guaranteedPeriod, Memory: &guaranteedMemory},
expected: &ResourceConfig{
CpuShares: &guaranteedShares,
CpuQuota: &guaranteedQuota,
CpuPeriod: &guaranteedPeriod,
Memory: &guaranteedMemory,
},
},
}
for testName, testCase := range testCases {
Expand All @@ -145,55 +206,65 @@ func TestResourceConfigForPod(t *testing.T) {

func TestMilliCPUToQuota(t *testing.T) {
testCases := []struct {
input int64
quota int64
cpu int64
period int64
quota int64
}{
{
input: int64(0),
cpu: int64(0),
period: int64(100000),
quota: int64(0),
period: int64(0),
},
{
input: int64(5),
quota: int64(1000),
cpu: int64(5),
period: int64(100000),
quota: int64(1000),
},
{
input: int64(9),
quota: int64(1000),
cpu: int64(9),
period: int64(100000),
quota: int64(1000),
},
{
input: int64(10),
quota: int64(1000),
cpu: int64(10),
period: int64(100000),
quota: int64(1000),
},
{
input: int64(200),
quota: int64(20000),
cpu: int64(200),
period: int64(100000),
quota: int64(20000),
},
{
input: int64(500),
quota: int64(50000),
cpu: int64(500),
period: int64(100000),
quota: int64(50000),
},
{
input: int64(1000),
quota: int64(100000),
cpu: int64(1000),
period: int64(100000),
quota: int64(100000),
},
{
input: int64(1500),
quota: int64(150000),
cpu: int64(1500),
period: int64(100000),
quota: int64(150000),
},
{
cpu: int64(1500),
period: int64(10000),
quota: int64(15000),
},
{
cpu: int64(250),
period: int64(5000),
quota: int64(1250),
},
}
for _, testCase := range testCases {
quota, period := MilliCPUToQuota(testCase.input)
if quota != testCase.quota || period != testCase.period {
t.Errorf("Input %v, expected quota %v period %v, but got quota %v period %v", testCase.input, testCase.quota, testCase.period, quota, period)
quota := MilliCPUToQuota(testCase.cpu, testCase.period)
if quota != testCase.quota {
t.Errorf("Input (cpu=%d, period=%d), expected quota=%d but got quota=%d", testCase.cpu, testCase.period, testCase.quota, quota)
}
}
}
15 changes: 7 additions & 8 deletions pkg/kubelet/kuberuntime/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ const (
milliCPUToCPU = 1000

// 100000 is equivalent to 100ms
quotaPeriod = 100 * minQuotaPeriod
minQuotaPeriod = 1000
defaultQuotaPeriod int64 = 100000
minQuotaPeriod int64 = 1000
)

var (
Expand Down Expand Up @@ -176,22 +176,21 @@ func milliCPUToShares(milliCPU int64) int64 {
return shares
}

// milliCPUToQuota converts milliCPU to CFS quota and period values
func milliCPUToQuota(milliCPU int64) (quota int64, period int64) {
// milliCPUToQuota takes milliCPU (along with a CFS period, in usec) and returns
// a CFS quota value
func milliCPUToQuota(milliCPU, period int64) (quota int64) {
// CFS quota is measured in two values:
// - cfs_period_us=100ms (the amount of time to measure usage across)
// - cfs_quota=20ms (the amount of cpu time allowed to be used across a period)
// so in the above example, you are limited to 20% of a single CPU
// for multi-cpu environments, you just scale equivalent amounts

if milliCPU == 0 {
return
}

// we set the period to 100ms by default
period = quotaPeriod

// we then convert your milliCPU to a value normalized over a period
quota = (milliCPU * quotaPeriod) / milliCPUToCPU
quota = (milliCPU * period) / milliCPUToCPU

// quota needs to be a minimum of 1ms.
if quota < minQuotaPeriod {
Expand Down
6 changes: 5 additions & 1 deletion pkg/kubelet/kuberuntime/kuberuntime_container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,9 +256,13 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.C
lc.Resources.OomScoreAdj = oomScoreAdj

if m.cpuCFSQuota {
cpuPeriod := defaultQuotaPeriod
if period, found := container.Resources.Limits[v1.ResourceCPUPeriodUsec]; found {
cpuPeriod = period.Value()
}
// if cpuLimit.Amount is nil, then the appropriate default value is returned
// to allow full usage of cpu resource.
cpuQuota, cpuPeriod := milliCPUToQuota(cpuLimit.MilliValue())
cpuQuota := milliCPUToQuota(cpuLimit.MilliValue(), cpuPeriod)
lc.Resources.CpuQuota = cpuQuota
lc.Resources.CpuPeriod = cpuPeriod
}
Expand Down
7 changes: 7 additions & 0 deletions staging/src/k8s.io/api/core/v1/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3599,6 +3599,13 @@ const (
// NVIDIA GPU, in devices. Alpha, might change: although fractional and allowing values >1, only one whole device per node is assigned.
ResourceNvidiaGPU ResourceName = "alpha.kubernetes.io/nvidia-gpu"
// Number of Pods that may be running on this Node: see ResourcePods

// -- Monzo-specific

// CPU throttling period, in microseconds.
// NOTE: Only set this value yourself if you know very well how to tune the
// Linux CFS scheduler, or in consultation with the Platform team.
ResourceCPUPeriodUsec ResourceName = "monzo.com/cpu-period"
)

const (
Expand Down