Account for kernel reserved memory in capacity calculations

cluster-autoscaler/cloudprovider/gce/gce_manager.go

@@ -26,6 +26,7 @@ import (
 
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/units"
 
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -478,7 +479,7 @@ func (m *gceManagerImpl) getCpuAndMemoryForMachineType(machineType string, zone
 		}
 		m.cache.AddMachineToCache(machineType, zone, machine)
 	}
-	return machine.GuestCpus, machine.MemoryMb * bytesPerMB, nil
+	return machine.GuestCpus, machine.MemoryMb * units.MiB, nil
 }
 
 func parseCustomMachineType(machineType string) (cpu, mem int64, err error) {
@@ -492,6 +493,6 @@ func parseCustomMachineType(machineType string) (cpu, mem int64, err error) {
 		return 0, 0, fmt.Errorf("failed to parse all params in %s", machineType)
 	}
 	// Mb to bytes
-	mem = mem * bytesPerMB
+	mem = mem * units.MiB
 	return
 }

cluster-autoscaler/cloudprovider/gce/gce_manager_test.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/units"
 
 	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
 
@@ -1092,29 +1093,29 @@ func TestGetCpuAndMemoryForMachineType(t *testing.T) {
 	cpu, mem, err := g.getCpuAndMemoryForMachineType("custom-8-2", zoneB)
 	assert.NoError(t, err)
 	assert.Equal(t, int64(8), cpu)
-	assert.Equal(t, int64(2*bytesPerMB), mem)
+	assert.Equal(t, int64(2*units.MiB), mem)
 	mock.AssertExpectationsForObjects(t, server)
 
 	// Standard machine type found in cache.
 	cpu, mem, err = g.getCpuAndMemoryForMachineType("n1-standard-1", zoneB)
 	assert.NoError(t, err)
 	assert.Equal(t, int64(1), cpu)
-	assert.Equal(t, int64(1*bytesPerMB), mem)
+	assert.Equal(t, int64(1*units.MiB), mem)
 	mock.AssertExpectationsForObjects(t, server)
 
 	// Standard machine type not found in cache.
 	server.On("handle", "/project1/zones/"+zoneB+"/machineTypes/n1-standard-2").Return(getMachineTypeResponse).Once()
 	cpu, mem, err = g.getCpuAndMemoryForMachineType("n1-standard-2", zoneB)
 	assert.NoError(t, err)
 	assert.Equal(t, int64(2), cpu)
-	assert.Equal(t, int64(3840*bytesPerMB), mem)
+	assert.Equal(t, int64(3840*units.MiB), mem)
 	mock.AssertExpectationsForObjects(t, server)
 
 	// Standard machine type cached.
 	cpu, mem, err = g.getCpuAndMemoryForMachineType("n1-standard-2", zoneB)
 	assert.NoError(t, err)
 	assert.Equal(t, int64(2), cpu)
-	assert.Equal(t, int64(3840*bytesPerMB), mem)
+	assert.Equal(t, int64(3840*units.MiB), mem)
 	mock.AssertExpectationsForObjects(t, server)
 
 	// Standard machine type not found in the zone.
@@ -1129,7 +1130,7 @@ func TestParseCustomMachineType(t *testing.T) {
 	cpu, mem, err := parseCustomMachineType("custom-2-2816")
 	assert.NoError(t, err)
 	assert.Equal(t, int64(2), cpu)
-	assert.Equal(t, int64(2816*bytesPerMB), mem)
+	assert.Equal(t, int64(2816*units.MiB), mem)
 	cpu, mem, err = parseCustomMachineType("other-a2-2816")
 	assert.Error(t, err)
 	cpu, mem, err = parseCustomMachineType("other-2-2816")

cluster-autoscaler/cloudprovider/gce/gce_price_model.go

@@ -147,7 +147,7 @@ func getBasePrice(resources apiv1.ResourceList, startTime time.Time, endTime tim
 	cpu := resources[apiv1.ResourceCPU]
 	mem := resources[apiv1.ResourceMemory]
 	price += float64(cpu.MilliValue()) / 1000.0 * cpuPricePerHour * hours
-	price += float64(mem.Value()) / float64(units.Gigabyte) * memoryPricePerHourPerGb * hours
+	price += float64(mem.Value()) / float64(units.GiB) * memoryPricePerHourPerGb * hours
 	return price
 }
 

cluster-autoscaler/cloudprovider/gce/gce_price_model_test.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
 	. "k8s.io/autoscaler/cluster-autoscaler/utils/test"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/units"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -46,35 +47,35 @@ func TestGetNodePrice(t *testing.T) {
 	now := time.Now()
 
 	// regular
-	node1 := BuildTestNode("sillyname1", 8000, 30*1024*1024*1024)
+	node1 := BuildTestNode("sillyname1", 8000, 30*units.GiB)
 	node1.Labels = labels1
 	price1, err := model.NodePrice(node1, now, now.Add(time.Hour))
 	assert.NoError(t, err)
 
 	// preemptible
-	node2 := BuildTestNode("sillyname2", 8000, 30*1024*1024*1024)
+	node2 := BuildTestNode("sillyname2", 8000, 30*units.GiB)
 	node2.Labels = labels2
 	price2, err := model.NodePrice(node2, now, now.Add(time.Hour))
 	assert.NoError(t, err)
 	// preemptible nodes should be way cheaper than regular.
 	assert.True(t, price1 > 3*price2)
 
 	// custom node
-	node3 := BuildTestNode("sillyname3", 8000, 30*1024*1024*1024)
+	node3 := BuildTestNode("sillyname3", 8000, 30*units.GiB)
 	price3, err := model.NodePrice(node3, now, now.Add(time.Hour))
 	assert.NoError(t, err)
 	// custom nodes should be slightly more expensive than regular.
 	assert.True(t, price1 < price3)
 	assert.True(t, price1*1.2 > price3)
 
 	// regular with gpu
-	node4 := BuildTestNode("sillyname4", 8000, 30*1024*1024*1024)
+	node4 := BuildTestNode("sillyname4", 8000, 30*units.GiB)
 	node4.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(1, resource.DecimalSI)
 	node4.Labels = labels1
 	price4, err := model.NodePrice(node4, now, now.Add(time.Hour))
 
 	// preemptible with gpu
-	node5 := BuildTestNode("sillyname5", 8000, 30*1024*1024*1024)
+	node5 := BuildTestNode("sillyname5", 8000, 30*units.GiB)
 	node5.Labels = labels2
 	node5.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(1, resource.DecimalSI)
 	price5, err := model.NodePrice(node5, now, now.Add(time.Hour))
@@ -86,16 +87,16 @@ func TestGetNodePrice(t *testing.T) {
 	assert.True(t, price4 > 2*price1)
 
 	// small custom node
-	node6 := BuildTestNode("sillyname6", 1000, 3750*1024*1024)
+	node6 := BuildTestNode("sillyname6", 1000, 3750*units.MiB)
 	price6, err := model.NodePrice(node6, now, now.Add(time.Hour))
 	assert.NoError(t, err)
 	// 8 times smaller node should be 8 times less expensive.
 	assert.True(t, math.Abs(price3-8*price6) < 0.1)
 }
 
 func TestGetPodPrice(t *testing.T) {
-	pod1 := BuildTestPod("a1", 100, 500*1024*1024)
-	pod2 := BuildTestPod("a2", 2*100, 2*500*1024*1024)
+	pod1 := BuildTestPod("a1", 100, 500*units.MiB)
+	pod2 := BuildTestPod("a2", 2*100, 2*500*units.MiB)
 
 	model := &GcePriceModel{}
 	now := time.Now()

cluster-autoscaler/cloudprovider/gce/reserved.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+// There should be no imports as it is used standalone in e2e tests
+
+const (
+	// MiB - MebiByte size (2^20)
+	MiB = 1024 * 1024
+	// GiB - GibiByte size (2^30)
+	GiB = 1024 * 1024 * 1024
+
+	// KubeletEvictionHardMemory is subtracted from capacity
+	// when calculating allocatable (on top of kube-reserved).
+	// Equals kubelet "evictionHard: {memory.available}"
+	// We don't have a good place to get it from, but it has been hard-coded
+	// to 100Mi since at least k8s 1.4.
+	KubeletEvictionHardMemory = 100 * MiB
+
+	// Kernel reserved memory is subtracted when calculating total memory.
+	kernelReservedRatio  = 64
+	kernelReservedMemory = 16 * MiB
+	// Reserved memory for software IO TLB
+	swiotlbReservedMemory  = 64 * MiB
+	swiotlbThresholdMemory = 3 * GiB
+)
+
+// CalculateKernelReserved computes how much memory Linux kernel will reserve.
+// TODO(jkaniuk): account for crashkernel reservation on RHEL / CentOS
+func CalculateKernelReserved(physicalMemory int64) int64 {
+	// Account for memory reserved by kernel
+	reserved := int64(physicalMemory / kernelReservedRatio)
+	reserved += kernelReservedMemory
+	// Account for software IO TLB allocation if memory requires 64bit addressing
+	if physicalMemory > swiotlbThresholdMemory {
+		reserved += swiotlbReservedMemory
+	}
+	return reserved
+}

cluster-autoscaler/cloudprovider/gce/reserved_test.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCalculateKernelReserved(t *testing.T) {
+	type testCase struct {
+		physicalMemory int64
+		reservedMemory int64
+	}
+	testCases := []testCase{
+		{
+			physicalMemory: 256 * MiB,
+			reservedMemory: 4*MiB + kernelReservedMemory,
+		},
+		{
+			physicalMemory: 2 * GiB,
+			reservedMemory: 32*MiB + kernelReservedMemory,
+		},
+		{
+			physicalMemory: 3 * GiB,
+			reservedMemory: 48*MiB + kernelReservedMemory,
+		},
+		{
+			physicalMemory: 3.25 * GiB,
+			reservedMemory: 52*MiB + kernelReservedMemory + swiotlbReservedMemory,
+		},
+		{
+			physicalMemory: 4 * GiB,
+			reservedMemory: 64*MiB + kernelReservedMemory + swiotlbReservedMemory,
+		},
+		{
+			physicalMemory: 128 * GiB,
+			reservedMemory: 2*GiB + kernelReservedMemory + swiotlbReservedMemory,
+		},
+	}
+	for idx, tc := range testCases {
+		t.Run(fmt.Sprintf("%v", idx), func(t *testing.T) {
+			reserved := CalculateKernelReserved(tc.physicalMemory)
+			assert.Equal(t, tc.reservedMemory, reserved)
+		})
+	}
+}