diff --git a/cluster-autoscaler/cloudprovider/gce/gce_manager.go b/cluster-autoscaler/cloudprovider/gce/gce_manager.go index 107253920019..4a6389c37f3b 100644 --- a/cluster-autoscaler/cloudprovider/gce/gce_manager.go +++ b/cluster-autoscaler/cloudprovider/gce/gce_manager.go @@ -119,6 +119,7 @@ type gceManagerImpl struct { regional bool explicitlyConfigured map[GceRef]bool migAutoDiscoverySpecs []migAutoDiscoveryConfig + reserved *GceReserved } // CreateGceManager constructs GceManager object. @@ -188,6 +189,7 @@ func CreateGceManager(configReader io.Reader, discoveryOpts cloudprovider.NodeGr interrupt: make(chan struct{}), explicitlyConfigured: make(map[GceRef]bool), concurrentGceRefreshes: concurrentGceRefreshes, + reserved: &GceReserved{}, } if err := manager.fetchExplicitMigs(discoveryOpts.NodeGroupSpecs); err != nil { @@ -586,7 +588,7 @@ func (m *gceManagerImpl) GetMigTemplateNode(mig Mig) (*apiv1.Node, error) { if err != nil { return nil, err } - return m.templates.BuildNodeFromTemplate(mig, template, cpu, mem, nil) + return m.templates.BuildNodeFromTemplate(mig, template, cpu, mem, nil, m.reserved) } func (m *gceManagerImpl) getCpuAndMemoryForMachineType(machineType string, zone string) (cpu int64, mem int64, err error) { diff --git a/cluster-autoscaler/cloudprovider/gce/reserved.go b/cluster-autoscaler/cloudprovider/gce/gce_reserved.go similarity index 96% rename from cluster-autoscaler/cloudprovider/gce/reserved.go rename to cluster-autoscaler/cloudprovider/gce/gce_reserved.go index 4d901d3ee689..d5dabbcb114e 100644 --- a/cluster-autoscaler/cloudprovider/gce/reserved.go +++ b/cluster-autoscaler/cloudprovider/gce/gce_reserved.go @@ -80,9 +80,12 @@ type EvictionHard struct { EphemeralStorageEvictionRatio float64 } +// GceReserved implement Reserved interface +type GceReserved struct{} + // CalculateKernelReserved computes how much memory Linux kernel will reserve. // TODO(jkaniuk): account for crashkernel reservation on RHEL / CentOS -func CalculateKernelReserved(physicalMemory int64, os OperatingSystem, osDistribution OperatingSystemDistribution) int64 { +func (r *GceReserved) CalculateKernelReserved(physicalMemory int64, os OperatingSystem, osDistribution OperatingSystemDistribution, nodeVersion string) int64 { switch os { case OperatingSystemLinux: // Account for memory reserved by kernel @@ -269,7 +272,7 @@ func EphemeralStorageOnLocalSSDFilesystemOverheadInBytes(diskCount int64, osDist } // CalculateOSReservedEphemeralStorage estimates how much ephemeral storage OS will reserve and eviction threshold -func CalculateOSReservedEphemeralStorage(diskSize int64, osDistribution OperatingSystemDistribution) int64 { +func (r *GceReserved) CalculateOSReservedEphemeralStorage(diskSize int64, os OperatingSystem, osDistribution OperatingSystemDistribution, nodeVersion string) int64 { switch osDistribution { case OperatingSystemDistributionCOS, OperatingSystemDistributionCOSContainerd: storage := int64(math.Ceil(0.015635*float64(diskSize))) + int64(math.Ceil(4.148*GiB)) // os partition estimation diff --git a/cluster-autoscaler/cloudprovider/gce/reserved_test.go b/cluster-autoscaler/cloudprovider/gce/gce_reserved_test.go similarity index 97% rename from cluster-autoscaler/cloudprovider/gce/reserved_test.go rename to cluster-autoscaler/cloudprovider/gce/gce_reserved_test.go index 74b3b58b3898..07b2d68c1043 100644 --- a/cluster-autoscaler/cloudprovider/gce/reserved_test.go +++ b/cluster-autoscaler/cloudprovider/gce/gce_reserved_test.go @@ -93,8 +93,9 @@ func TestCalculateKernelReservedLinux(t *testing.T) { }, } for idx, tc := range testCases { + r := &GceReserved{} t.Run(fmt.Sprintf("%v", idx), func(t *testing.T) { - reserved := CalculateKernelReserved(tc.physicalMemory, OperatingSystemLinux, tc.osDistribution) + reserved := r.CalculateKernelReserved(tc.physicalMemory, OperatingSystemLinux, tc.osDistribution, "") if tc.osDistribution == OperatingSystemDistributionUbuntu { assert.Equal(t, tc.reservedMemory+int64(math.Min(correctionConstant*float64(tc.physicalMemory), maximumCorrectionValue)+ubuntuSpecificOffset), reserved) } else if tc.osDistribution == OperatingSystemDistributionCOS { diff --git a/cluster-autoscaler/cloudprovider/gce/os_reserved.go b/cluster-autoscaler/cloudprovider/gce/os_reserved.go new file mode 100644 index 000000000000..f39657954f8f --- /dev/null +++ b/cluster-autoscaler/cloudprovider/gce/os_reserved.go @@ -0,0 +1,28 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +// OsReservedCalculator calculates the OS reserved values. +type OsReservedCalculator interface { + // CalculateKernelReserved computes how much memory OS kernel will reserve. + // NodeVersion parameter is optional. If empty string is passed a result calculated using default node version will be returned. + CalculateKernelReserved(physicalMemory int64, os OperatingSystem, osDistribution OperatingSystemDistribution, nodeVersion string) int64 + + // CalculateOSReservedEphemeralStorage estimates how much ephemeral storage OS will reserve and eviction threshold. + // NodeVersion parameter may be optional if it is not used in calculation. + CalculateOSReservedEphemeralStorage(diskSize int64, os OperatingSystem, osDistribution OperatingSystemDistribution, nodeVersion string) int64 +} diff --git a/cluster-autoscaler/cloudprovider/gce/templates.go b/cluster-autoscaler/cloudprovider/gce/templates.go index b8c6e95ed10e..6c04be2d0184 100644 --- a/cluster-autoscaler/cloudprovider/gce/templates.go +++ b/cluster-autoscaler/cloudprovider/gce/templates.go @@ -59,7 +59,7 @@ func (t *GceTemplateBuilder) getAcceleratorCount(accelerators []*gce.Accelerator } // BuildCapacity builds a list of resource capacities given list of hardware. -func (t *GceTemplateBuilder) BuildCapacity(cpu int64, mem int64, accelerators []*gce.AcceleratorConfig, os OperatingSystem, osDistribution OperatingSystemDistribution, ephemeralStorage int64, ephemeralStorageLocalSSDCount int64, pods *int64) (apiv1.ResourceList, error) { +func (t *GceTemplateBuilder) BuildCapacity(cpu int64, mem int64, accelerators []*gce.AcceleratorConfig, os OperatingSystem, osDistribution OperatingSystemDistribution, ephemeralStorage int64, ephemeralStorageLocalSSDCount int64, pods *int64, r OsReservedCalculator) (apiv1.ResourceList, error) { capacity := apiv1.ResourceList{} if pods == nil { capacity[apiv1.ResourcePods] = *resource.NewQuantity(110, resource.DecimalSI) @@ -68,7 +68,7 @@ func (t *GceTemplateBuilder) BuildCapacity(cpu int64, mem int64, accelerators [] } capacity[apiv1.ResourceCPU] = *resource.NewQuantity(cpu, resource.DecimalSI) - memTotal := mem - CalculateKernelReserved(mem, os, osDistribution) + memTotal := mem - r.CalculateKernelReserved(mem, os, osDistribution, "") capacity[apiv1.ResourceMemory] = *resource.NewQuantity(memTotal, resource.DecimalSI) if accelerators != nil && len(accelerators) > 0 { @@ -80,7 +80,7 @@ func (t *GceTemplateBuilder) BuildCapacity(cpu int64, mem int64, accelerators [] if ephemeralStorageLocalSSDCount > 0 { storageTotal = ephemeralStorage - EphemeralStorageOnLocalSSDFilesystemOverheadInBytes(ephemeralStorageLocalSSDCount, osDistribution) } else { - storageTotal = ephemeralStorage - CalculateOSReservedEphemeralStorage(ephemeralStorage, osDistribution) + storageTotal = ephemeralStorage - r.CalculateOSReservedEphemeralStorage(ephemeralStorage, os, osDistribution, "") } capacity[apiv1.ResourceEphemeralStorage] = *resource.NewQuantity(int64(math.Max(float64(storageTotal), 0)), resource.DecimalSI) } @@ -144,7 +144,7 @@ func getKubeEnvValueFromTemplateMetadata(template *gce.InstanceTemplate) (string } // BuildNodeFromTemplate builds node from provided GCE template. -func (t *GceTemplateBuilder) BuildNodeFromTemplate(mig Mig, template *gce.InstanceTemplate, cpu int64, mem int64, pods *int64) (*apiv1.Node, error) { +func (t *GceTemplateBuilder) BuildNodeFromTemplate(mig Mig, template *gce.InstanceTemplate, cpu int64, mem int64, pods *int64, reserved OsReservedCalculator) (*apiv1.Node, error) { if template.Properties == nil { return nil, fmt.Errorf("instance template %s has no properties", template.Name) @@ -186,7 +186,7 @@ func (t *GceTemplateBuilder) BuildNodeFromTemplate(mig Mig, template *gce.Instan return nil, fmt.Errorf("could not fetch ephemeral storage from instance template: %v", err) } - capacity, err := t.BuildCapacity(cpu, mem, template.Properties.GuestAccelerators, os, osDistribution, ephemeralStorage, ssdCount, pods) + capacity, err := t.BuildCapacity(cpu, mem, template.Properties.GuestAccelerators, os, osDistribution, ephemeralStorage, ssdCount, pods, reserved) if err != nil { return nil, err } diff --git a/cluster-autoscaler/cloudprovider/gce/templates_test.go b/cluster-autoscaler/cloudprovider/gce/templates_test.go index e3a944ab1aec..6dbfc82f427b 100644 --- a/cluster-autoscaler/cloudprovider/gce/templates_test.go +++ b/cluster-autoscaler/cloudprovider/gce/templates_test.go @@ -221,7 +221,7 @@ func TestBuildNodeFromTemplateSetsResources(t *testing.T) { if tc.kubeEnv != "" { template.Properties.Metadata.Items = []*gce.MetadataItems{{Key: "kube-env", Value: &tc.kubeEnv}} } - node, err := tb.BuildNodeFromTemplate(mig, template, tc.physicalCpu, tc.physicalMemory, tc.pods) + node, err := tb.BuildNodeFromTemplate(mig, template, tc.physicalCpu, tc.physicalMemory, tc.pods, &GceReserved{}) if tc.expectedErr { assert.Error(t, err) } else { @@ -238,7 +238,7 @@ func TestBuildNodeFromTemplateSetsResources(t *testing.T) { } else if tc.isEphemeralStorageBlocked { physicalEphemeralStorageGiB = 0 } - capacity, err := tb.BuildCapacity(tc.physicalCpu, tc.physicalMemory, tc.accelerators, OperatingSystemLinux, OperatingSystemDistributionCOS, physicalEphemeralStorageGiB*units.GiB, tc.ephemeralStorageLocalSSDCount, tc.pods) + capacity, err := tb.BuildCapacity(tc.physicalCpu, tc.physicalMemory, tc.accelerators, OperatingSystemLinux, OperatingSystemDistributionCOS, physicalEphemeralStorageGiB*units.GiB, tc.ephemeralStorageLocalSSDCount, tc.pods, &GceReserved{}) assert.NoError(t, err) assertEqualResourceLists(t, "Capacity", capacity, node.Status.Capacity) if !tc.kubeReserved { @@ -532,7 +532,7 @@ func TestBuildCapacityMemory(t *testing.T) { t.Run(fmt.Sprintf("%v", idx), func(t *testing.T) { tb := GceTemplateBuilder{} noAccelerators := make([]*gce.AcceleratorConfig, 0) - buildCapacity, err := tb.BuildCapacity(tc.physicalCpu, tc.physicalMemory, noAccelerators, tc.os, OperatingSystemDistributionCOS, -1, 0, nil) + buildCapacity, err := tb.BuildCapacity(tc.physicalCpu, tc.physicalMemory, noAccelerators, tc.os, OperatingSystemDistributionCOS, -1, 0, nil, &GceReserved{}) assert.NoError(t, err) expectedCapacity, err := makeResourceList2(tc.physicalCpu, tc.expectedCapacityMemory, 0, 110) assert.NoError(t, err)