Enable more optional configurations for AKS node pools

api/v1beta1/types.go

@@ -698,3 +698,29 @@ type AzureBastion struct {
 func IsTerminalProvisioningState(state ProvisioningState) bool {
 	return state == Failed || state == Succeeded
 }
+
+// KubeletConfig kubelet configurations of agent nodes.
+type KubeletConfig struct {
+	// CPUManagerPolicy - CPU Manager policy to use.
+	CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"`
+	// CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits.
+	CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"`
+	// CPUCfsQuotaPeriod - Sets CPU CFS quota period value.
+	CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"`
+	// ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run.
+	ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"`
+	// ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run.
+	ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"`
+	// TopologyManagerPolicy - Topology Manager policy to use.
+	TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"`
+	// AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`).
+	AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"`
+	// FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node.
+	FailSwapOn *bool `json:"failSwapOn,omitempty"`
+	// ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated.
+	ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"`
+	// ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2.
+	ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"`
+	// PodMaxPids - The maximum number of processes per pod.
+	PodMaxPids *int32 `json:"podMaxPids,omitempty"`
+}

azure/scope/managedcontrolplane.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -504,6 +505,28 @@ func (s *ManagedControlPlaneScope) GetAgentPoolSpecs(ctx context.Context) ([]azu
 			ammp.Replicas = *ownerPool.Spec.Replicas
 		}
 
+		if pool.Spec.VnetSubnetID != nil {
+			ammp.VnetSubnetID = *pool.Spec.VnetSubnetID
+		}
+
+		if pool.Spec.KubeletConfig != nil {
+			ammp.KubeletConfig = (*infrav1.KubeletConfig)(pool.Spec.KubeletConfig)
+		}
+
+		if pool.Spec.AutoScaling != nil {
+			ammp.EnableAutoScaling = to.BoolPtr(true)
+			ammp.MaxCount = pool.Spec.AutoScaling.MaxCount
+			ammp.MinCount = pool.Spec.AutoScaling.MinCount
+		}
+		ammp.EnableFIPS = pool.Spec.EnableFIPS
+		ammp.EnableNodePublicIP = pool.Spec.EnableNodePublicIP
+		ammp.AvailabilityZones = pool.Spec.AvailabilityZones
+		ammp.NodeLabels = pool.Spec.NodeLabels
+		ammp.NodeTaints = pool.Spec.NodeTaints
+		ammp.OsDiskType = pool.Spec.OsDiskType
+		ammp.ScaleSetPriority = pool.Spec.ScaleSetPriority
+		ammp.MaxPods = pool.Spec.MaxPods
+
 		ammps = append(ammps, ammp)
 	}
 
@@ -547,6 +570,28 @@ func (s *ManagedControlPlaneScope) AgentPoolSpec() azure.AgentPoolSpec {
 		agentPoolSpec.OSDiskSizeGB = *s.InfraMachinePool.Spec.OSDiskSizeGB
 	}
 
+	if s.InfraMachinePool.Spec.VnetSubnetID != nil {
+		agentPoolSpec.VnetSubnetID = *s.InfraMachinePool.Spec.VnetSubnetID
+	}
+
+	if s.InfraMachinePool.Spec.KubeletConfig != nil {
+		agentPoolSpec.KubeletConfig = (*infrav1.KubeletConfig)(s.InfraMachinePool.Spec.KubeletConfig)
+	}
+
+	if s.InfraMachinePool.Spec.AutoScaling != nil {
+		agentPoolSpec.EnableAutoScaling = to.BoolPtr(true)
+		agentPoolSpec.MaxCount = s.InfraMachinePool.Spec.AutoScaling.MaxCount
+		agentPoolSpec.MinCount = s.InfraMachinePool.Spec.AutoScaling.MinCount
+	}
+	agentPoolSpec.EnableFIPS = s.InfraMachinePool.Spec.EnableFIPS
+	agentPoolSpec.EnableNodePublicIP = s.InfraMachinePool.Spec.EnableNodePublicIP
+	agentPoolSpec.NodeLabels = s.InfraMachinePool.Spec.NodeLabels
+	agentPoolSpec.NodeTaints = s.InfraMachinePool.Spec.NodeTaints
+	agentPoolSpec.OsDiskType = s.InfraMachinePool.Spec.OsDiskType
+	agentPoolSpec.AvailabilityZones = s.InfraMachinePool.Spec.AvailabilityZones
+	agentPoolSpec.ScaleSetPriority = s.InfraMachinePool.Spec.ScaleSetPriority
+	agentPoolSpec.MaxPods = s.InfraMachinePool.Spec.MaxPods
+
 	return agentPoolSpec
 }
 

azure/services/agentpools/agentpools.go

@@ -80,6 +80,28 @@ func (s *Service) Reconcile(ctx context.Context) error {
 		},
 	}
 
+	if agentPoolSpec.EnableAutoScaling != nil {
+		profile.EnableAutoScaling = agentPoolSpec.EnableAutoScaling
+		profile.MaxCount = agentPoolSpec.MaxCount
+		profile.MinCount = agentPoolSpec.MinCount
+	}
+
+	profile.EnableFIPS = agentPoolSpec.EnableFIPS
+	profile.EnableNodePublicIP = agentPoolSpec.EnableNodePublicIP
+	profile.NodeLabels = agentPoolSpec.NodeLabels
+	profile.NodeTaints = &agentPoolSpec.NodeTaints
+	profile.AvailabilityZones = &agentPoolSpec.AvailabilityZones
+	profile.MaxPods = agentPoolSpec.MaxPods
+	if agentPoolSpec.OsDiskType != nil {
+		profile.OsDiskType = containerservice.OSDiskType(*agentPoolSpec.OsDiskType)
+	}
+	if agentPoolSpec.ScaleSetPriority != nil {
+		profile.ScaleSetPriority = containerservice.ScaleSetPriority(*agentPoolSpec.ScaleSetPriority)
+	}
+	if agentPoolSpec.KubeletConfig != nil {
+		profile.KubeletConfig = (*containerservice.KubeletConfig)(agentPoolSpec.KubeletConfig)
+	}
+
 	existingPool, err := s.Client.Get(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name)
 	if err != nil && !azure.ResourceNotFound(err) {
 		return errors.Wrap(err, "failed to get existing agent pool")
@@ -109,6 +131,10 @@ func (s *Service) Reconcile(ctx context.Context) error {
 				Count:               existingPool.Count,
 				OrchestratorVersion: existingPool.OrchestratorVersion,
 				Mode:                existingPool.Mode,
+				MaxCount:            existingPool.MaxCount,
+				MinCount:            existingPool.MinCount,
+				EnableAutoScaling:   existingPool.EnableAutoScaling,
+				NodeLabels:          existingPool.NodeLabels,
 			},
 		}
 
@@ -117,6 +143,10 @@ func (s *Service) Reconcile(ctx context.Context) error {
 				Count:               profile.Count,
 				OrchestratorVersion: profile.OrchestratorVersion,
 				Mode:                profile.Mode,
+				MaxCount:            profile.MaxCount,
+				MinCount:            profile.MinCount,
+				EnableAutoScaling:   profile.EnableAutoScaling,
+				NodeLabels:          profile.NodeLabels,
 			},
 		}
 

azure/services/managedclusters/managedclusters.go

@@ -145,6 +145,31 @@ func New(scope ManagedClusterScope) *Service {
 	}
 }
 
+// SetAgentPool set the ManagedClusterAgentPoolProfile values.
+func SetAgentPool(profile *containerservice.ManagedClusterAgentPoolProfile, pool *azure.AgentPoolSpec) {
+	if pool.OsDiskType != nil {
+		profile.OsDiskType = containerservice.OSDiskType(*pool.OsDiskType)
+	}
+
+	if pool.ScaleSetPriority != nil {
+		profile.ScaleSetPriority = containerservice.ScaleSetPriority(*pool.ScaleSetPriority)
+	}
+
+	if pool.KubeletConfig != nil {
+		profile.KubeletConfig = (*containerservice.KubeletConfig)(pool.KubeletConfig)
+	}
+
+	profile.MaxCount = pool.MaxCount
+	profile.MinCount = pool.MinCount
+	profile.EnableAutoScaling = pool.EnableAutoScaling
+	profile.EnableFIPS = pool.EnableFIPS
+	profile.EnableNodePublicIP = pool.EnableNodePublicIP
+	profile.NodeLabels = pool.NodeLabels
+	profile.NodeTaints = &pool.NodeTaints
+	profile.AvailabilityZones = &pool.AvailabilityZones
+	profile.MaxPods = pool.MaxPods
+}
+
 // Reconcile idempotently creates or updates a managed cluster, if possible.
 func (s *Service) Reconcile(ctx context.Context) error {
 	ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.Service.Reconcile")
@@ -239,9 +264,15 @@ func (s *Service) Reconcile(ctx context.Context) error {
 			OsDiskSizeGB: &pool.OSDiskSizeGB,
 			Count:        &pool.Replicas,
 			Type:         containerservice.AgentPoolTypeVirtualMachineScaleSets,
-			VnetSubnetID: &managedClusterSpec.VnetSubnetID,
 			Mode:         containerservice.AgentPoolMode(pool.Mode),
 		}
+		if pool.VnetSubnetID != "" {
+			profile.VnetSubnetID = &pool.VnetSubnetID
+		} else {
+			profile.VnetSubnetID = &managedClusterSpec.VnetSubnetID
+		}
+
+		SetAgentPool(&profile, &pool)
 		*managedCluster.AgentPoolProfiles = append(*managedCluster.AgentPoolProfiles, profile)
 	}
 

azure/types.go

@@ -434,4 +434,40 @@ type AgentPoolSpec struct {
 
 	// Mode represents mode of an agent pool. Possible values include: 'System', 'User'.
 	Mode string
+
+	// Max count for auto scaling
+	MaxCount *int32 `json:"maxCount,omitempty"`
+
+	// Min count for auto scaling
+	MinCount *int32 `json:"minCount,omitempty"`
+
+	// Enable auto scaling
+	EnableAutoScaling *bool `json:"EnableAutoScaling,omitempty"`
+
+	// Enable FIPS node image
+	EnableFIPS *bool `json:"EnableFIPS,omitempty"`
+
+	// Enable node public IP
+	EnableNodePublicIP *bool `json:"EnableNodePublicIP,omitempty"`
+
+	// Node labels
+	NodeLabels map[string]*string `json:"NodeLabels,omitempty"`
+
+	// Node taints
+	NodeTaints []string `json:"NodeTaints,omitempty"`
+
+	// Node OS disk type
+	OsDiskType *string `json:"OsDiskType,omitempty"`
+
+	// AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType.
+	AvailabilityZones []string `json:"availabilityZones,omitempty"`
+
+	// ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular'
+	ScaleSetPriority *string `json:"scaleSetPriority,omitempty"`
+
+	// MaxPods - Maximum number of pods that can run on a node.
+	MaxPods *int32 `json:"maxPods,omitempty"`
+
+	// KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes.
+	KubeletConfig *infrav1.KubeletConfig `json:"kubeletConfig,omitempty"`
 }