feat: AKS node pool KubeletConfig

azure/converters/managedagentpool.go

@@ -23,7 +23,7 @@ import (
 // AgentPoolToManagedClusterAgentPoolProfile converts a AgentPoolSpec to an Azure SDK ManagedClusterAgentPoolProfile used in managedcluster reconcile.
 func AgentPoolToManagedClusterAgentPoolProfile(pool containerservice.AgentPool) containerservice.ManagedClusterAgentPoolProfile {
 	properties := pool.ManagedClusterAgentPoolProfileProperties
-	return containerservice.ManagedClusterAgentPoolProfile{
+	agentPool := containerservice.ManagedClusterAgentPoolProfile{
 		Name:                 pool.Name, // Note: if converting from agentPoolSpec.Parameters(), this field will not be set
 		VMSize:               properties.VMSize,
 		OsType:               properties.OsType,
@@ -46,4 +46,8 @@ func AgentPoolToManagedClusterAgentPoolProfile(pool containerservice.AgentPool)
 		NodePublicIPPrefixID: properties.NodePublicIPPrefixID,
 		ScaleSetPriority:     properties.ScaleSetPriority,
 	}
+	if properties.KubeletConfig != nil {
+		agentPool.KubeletConfig = properties.KubeletConfig
+	}
+	return agentPool
 }

azure/scope/managedmachinepool.go

@@ -205,6 +205,24 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane
 		}
 	}
 
+	if managedMachinePool.Spec.KubeletConfig != nil {
+		agentPoolSpec.KubeletConfig = &agentpools.KubeletConfig{
+			CPUManagerPolicy:      managedMachinePool.Spec.KubeletConfig.CPUManagerPolicy,
+			CPUCfsQuota:           managedMachinePool.Spec.KubeletConfig.CPUCfsQuota,
+			CPUCfsQuotaPeriod:     managedMachinePool.Spec.KubeletConfig.CPUCfsQuotaPeriod,
+			ImageGcHighThreshold:  managedMachinePool.Spec.KubeletConfig.ImageGcHighThreshold,
+			ImageGcLowThreshold:   managedMachinePool.Spec.KubeletConfig.ImageGcLowThreshold,
+			TopologyManagerPolicy: managedMachinePool.Spec.KubeletConfig.TopologyManagerPolicy,
+			FailSwapOn:            managedMachinePool.Spec.KubeletConfig.FailSwapOn,
+			ContainerLogMaxSizeMB: managedMachinePool.Spec.KubeletConfig.ContainerLogMaxSizeMB,
+			ContainerLogMaxFiles:  managedMachinePool.Spec.KubeletConfig.ContainerLogMaxFiles,
+			PodMaxPids:            managedMachinePool.Spec.KubeletConfig.PodMaxPids,
+		}
+		if len(managedMachinePool.Spec.KubeletConfig.AllowedUnsafeSysctls) > 0 {
+			agentPoolSpec.KubeletConfig.AllowedUnsafeSysctls = &managedMachinePool.Spec.KubeletConfig.AllowedUnsafeSysctls
+		}
+	}
+
 	return agentPoolSpec
 }
 

azure/services/agentpools/spec.go

@@ -29,6 +29,32 @@ import (
 	azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure"
 )
 
+// KubeletConfig defines the set of kubelet configurations for nodes in pools.
+type KubeletConfig struct {
+	// CPUManagerPolicy - CPU Manager policy to use.
+	CPUManagerPolicy *string
+	// CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits.
+	CPUCfsQuota *bool
+	// CPUCfsQuotaPeriod - Sets CPU CFS quota period value.
+	CPUCfsQuotaPeriod *string
+	// ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run.
+	ImageGcHighThreshold *int32
+	// ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run.
+	ImageGcLowThreshold *int32
+	// TopologyManagerPolicy - Topology Manager policy to use.
+	TopologyManagerPolicy *string
+	// AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`).
+	AllowedUnsafeSysctls *[]string
+	// FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node.
+	FailSwapOn *bool
+	// ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated.
+	ContainerLogMaxSizeMB *int32
+	// ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2.
+	ContainerLogMaxFiles *int32
+	// PodMaxPids - The maximum number of processes per pod.
+	PodMaxPids *int32
+}
+
 // AgentPoolSpec contains agent pool specification details.
 type AgentPoolSpec struct {
 	// Name is the name of agent pool.
@@ -99,6 +125,9 @@ type AgentPoolSpec struct {
 
 	// ScaleSetPriority specifies the ScaleSetPriority for the node pool. Allowed values are 'Spot' and 'Regular'
 	ScaleSetPriority *string `json:"scaleSetPriority,omitempty"`
+
+	// KubeletConfig specifies the kubelet configurations for nodes.
+	KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"`
 }
 
 // ResourceName returns the name of the agent pool.
@@ -148,6 +177,7 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 				MaxCount:            existingPool.MaxCount,
 				NodeLabels:          existingPool.NodeLabels,
 				NodeTaints:          existingPool.NodeTaints,
+				KubeletConfig:       existingPool.KubeletConfig,
 			},
 		}
 
@@ -164,6 +194,22 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 			},
 		}
 
+		if s.KubeletConfig != nil {
+			normalizedProfile.KubeletConfig = &containerservice.KubeletConfig{
+				CPUManagerPolicy:      s.KubeletConfig.CPUManagerPolicy,
+				CPUCfsQuota:           s.KubeletConfig.CPUCfsQuota,
+				CPUCfsQuotaPeriod:     s.KubeletConfig.CPUCfsQuotaPeriod,
+				ImageGcHighThreshold:  s.KubeletConfig.ImageGcHighThreshold,
+				ImageGcLowThreshold:   s.KubeletConfig.ImageGcLowThreshold,
+				TopologyManagerPolicy: s.KubeletConfig.TopologyManagerPolicy,
+				FailSwapOn:            s.KubeletConfig.FailSwapOn,
+				ContainerLogMaxSizeMB: s.KubeletConfig.ContainerLogMaxSizeMB,
+				ContainerLogMaxFiles:  s.KubeletConfig.ContainerLogMaxFiles,
+				PodMaxPids:            s.KubeletConfig.PodMaxPids,
+				AllowedUnsafeSysctls:  s.KubeletConfig.AllowedUnsafeSysctls,
+			}
+		}
+
 		// When autoscaling is set, the count of the nodes differ based on the autoscaler and should not depend on the
 		// count present in MachinePool or AzureManagedMachinePool, hence we should not make an update API call based
 		// on difference in count.
@@ -202,12 +248,30 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 		vnetSubnetID = &s.VnetSubnetID
 	}
 
-	return containerservice.AgentPool{
+	var kubeletConfig *containerservice.KubeletConfig
+	if s.KubeletConfig != nil {
+		kubeletConfig = &containerservice.KubeletConfig{
+			CPUManagerPolicy:      s.KubeletConfig.CPUManagerPolicy,
+			CPUCfsQuota:           s.KubeletConfig.CPUCfsQuota,
+			CPUCfsQuotaPeriod:     s.KubeletConfig.CPUCfsQuotaPeriod,
+			ImageGcHighThreshold:  s.KubeletConfig.ImageGcHighThreshold,
+			ImageGcLowThreshold:   s.KubeletConfig.ImageGcLowThreshold,
+			TopologyManagerPolicy: s.KubeletConfig.TopologyManagerPolicy,
+			FailSwapOn:            s.KubeletConfig.FailSwapOn,
+			ContainerLogMaxSizeMB: s.KubeletConfig.ContainerLogMaxSizeMB,
+			ContainerLogMaxFiles:  s.KubeletConfig.ContainerLogMaxFiles,
+			PodMaxPids:            s.KubeletConfig.PodMaxPids,
+			AllowedUnsafeSysctls:  s.KubeletConfig.AllowedUnsafeSysctls,
+		}
+	}
+
+	agentPool := containerservice.AgentPool{
 		ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{
 			AvailabilityZones:    availabilityZones,
 			Count:                &s.Replicas,
 			EnableAutoScaling:    s.EnableAutoScaling,
 			EnableUltraSSD:       s.EnableUltraSSD,
+			KubeletConfig:        kubeletConfig,
 			MaxCount:             s.MaxCount,
 			MaxPods:              s.MaxPods,
 			MinCount:             s.MinCount,
@@ -225,7 +289,9 @@ func (s *AgentPoolSpec) Parameters(existing interface{}) (params interface{}, er
 			EnableNodePublicIP:   s.EnableNodePublicIP,
 			NodePublicIPPrefixID: s.NodePublicIPPrefixID,
 		},
-	}, nil
+	}
+
+	return agentPool, nil
 }
 
 // mergeSystemNodeLabels appends any kubernetes.azure.com-prefixed labels from the AKS label set

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml

@@ -213,6 +213,61 @@ spec:
                 description: EnableUltraSSD enables the storage type UltraSSD_LRS
                   for the agent pool.
                 type: boolean
+              kubeletConfig:
+                description: KubeletConfig specifies the kubelet configurations for
+                  nodes.
+                properties:
+                  allowedUnsafeSysctls:
+                    description: AllowedUnsafeSysctls - Allowlist of unsafe sysctls
+                      or unsafe sysctl patterns (ending in `*`).
+                    items:
+                      type: string
+                    type: array
+                  containerLogMaxFiles:
+                    description: ContainerLogMaxFiles - The maximum number of container
+                      log files that can be present for a container. The number must
+                      be ≥ 2.
+                    format: int32
+                    type: integer
+                  containerLogMaxSizeMB:
+                    description: ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi)
+                      of container log file before it is rotated.
+                    format: int32
+                    type: integer
+                  cpuCfsQuota:
+                    description: CPUCfsQuota - Enable CPU CFS quota enforcement for
+                      containers that specify CPU limits.
+                    type: boolean
+                  cpuCfsQuotaPeriod:
+                    description: CPUCfsQuotaPeriod - Sets CPU CFS quota period value.
+                    type: string
+                  cpuManagerPolicy:
+                    description: CPUManagerPolicy - CPU Manager policy to use.
+                    type: string
+                  failSwapOn:
+                    description: FailSwapOn - If set to true it will make the Kubelet
+                      fail to start if swap is enabled on the node.
+                    type: boolean
+                  imageGcHighThreshold:
+                    description: ImageGcHighThreshold - The percent of disk usage
+                      after which image garbage collection is always run.
+                    format: int32
+                    type: integer
+                  imageGcLowThreshold:
+                    description: ImageGcLowThreshold - The percent of disk usage before
+                      which image garbage collection is never run.
+                    format: int32
+                    type: integer
+                  podMaxPids:
+                    description: PodMaxPids - The maximum number of processes per
+                      pod.
+                    format: int32
+                    type: integer
+                  topologyManagerPolicy:
+                    description: TopologyManagerPolicy - Topology Manager policy to
+                      use.
+                    type: string
+                type: object
               maxPods:
                 description: MaxPods specifies the kubelet --max-pods configuration
                   for the node pool.

exp/api/v1alpha3/azuremanagedmachinepool_conversion.go

@@ -48,6 +48,9 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.EnableNodePublicIP = restored.Spec.EnableNodePublicIP
 	dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID
 	dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority
+	if restored.Spec.KubeletConfig != nil {
+		dst.Spec.KubeletConfig = restored.Spec.KubeletConfig
+	}
 
 	dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates
 	dst.Status.Conditions = restored.Status.Conditions

exp/api/v1alpha4/azuremanagedmachinepool_conversion.go

@@ -48,6 +48,9 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.EnableNodePublicIP = restored.Spec.EnableNodePublicIP
 	dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID
 	dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority
+	if restored.Spec.KubeletConfig != nil {
+		dst.Spec.KubeletConfig = restored.Spec.KubeletConfig
+	}
 
 	dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates
 	dst.Status.Conditions = restored.Status.Conditions

exp/api/v1beta1/azuremanagedmachinepool_types.go

@@ -41,6 +41,43 @@ const (
 // NodePoolMode enumerates the values for agent pool mode.
 type NodePoolMode string
 
+// KubeletConfig defines the set of kubelet configurations for nodes in pools.
+type KubeletConfig struct {
+	// CPUManagerPolicy - CPU Manager policy to use.
+	// +optional
+	CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"`
+	// CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits.
+	// +optional
+	CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"`
+	// CPUCfsQuotaPeriod - Sets CPU CFS quota period value.
+	// +optional
+	CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"`
+	// ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run.
+	// +optional
+	ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"`
+	// ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run.
+	// +optional
+	ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"`
+	// TopologyManagerPolicy - Topology Manager policy to use.
+	// +optional
+	TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"`
+	// AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`).
+	// +optional
+	AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"`
+	// FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node.
+	// +optional
+	FailSwapOn *bool `json:"failSwapOn,omitempty"`
+	// ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated.
+	// +optional
+	ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"`
+	// ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2.
+	// +optional
+	ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"`
+	// PodMaxPids - The maximum number of processes per pod.
+	// +optional
+	PodMaxPids *int32 `json:"podMaxPids,omitempty"`
+}
+
 // AzureManagedMachinePoolSpec defines the desired state of AzureManagedMachinePool.
 type AzureManagedMachinePoolSpec struct {
 
@@ -111,6 +148,10 @@ type AzureManagedMachinePoolSpec struct {
 	// +kubebuilder:validation:Enum=Regular;Spot
 	// +optional
 	ScaleSetPriority *string `json:"scaleSetPriority,omitempty"`
+
+	// KubeletConfig specifies the kubelet configurations for nodes.
+	// +optional
+	KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"`
 }
 
 // ManagedMachinePoolScaling specifies scaling options.