From f8181c4a173aed233e28d968b004925846832b42 Mon Sep 17 00:00:00 2001 From: Meixing Le Date: Mon, 20 Sep 2021 10:32:30 -0700 Subject: [PATCH] Enable more optional configurations for AKS node pools --- api/v1beta1/types.go | 26 ++ api/v1beta1/zz_generated.deepcopy.go | 74 +++++ azure/scope/managedcontrolplane.go | 45 +++ azure/services/agentpools/agentpools.go | 30 ++ .../managedclusters/managedclusters.go | 33 ++- azure/types.go | 36 +++ ...ter.x-k8s.io_azuremanagedmachinepools.yaml | 266 ++++++++++++++++++ .../azuremanagedmachinepool_conversion.go | 92 ++++++ exp/api/v1alpha3/zz_generated.conversion.go | 11 + .../v1alpha4/azuremanagedmachinepool_types.go | 110 ++++++++ exp/api/v1alpha4/zz_generated.conversion.go | 104 +++++++ exp/api/v1alpha4/zz_generated.deepcopy.go | 164 +++++++++++ .../v1beta1/azuremanagedmachinepool_types.go | 110 ++++++++ .../azuremanagedmachinepool_webhook.go | 243 +++++++++++++++- .../azuremanagedmachinepool_webhook_test.go | 208 ++++++++++++++ exp/api/v1beta1/zz_generated.deepcopy.go | 164 +++++++++++ 16 files changed, 1700 insertions(+), 16 deletions(-) diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index fe04ad2c466..c83c462221d 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -698,3 +698,29 @@ type AzureBastion struct { func IsTerminalProvisioningState(state ProvisioningState) bool { return state == Failed || state == Succeeded } + +// KubeletConfig kubelet configurations of agent nodes. +type KubeletConfig struct { + // CPUManagerPolicy - CPU Manager policy to use. + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + // CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits. + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + // TopologyManagerPolicy - Topology Manager policy to use. + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` + // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"` + // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // PodMaxPids - The maximum number of processes per pod. + PodMaxPids *int32 `json:"podMaxPids,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 33f9ddb57d3..bffdffe2e6e 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -861,6 +861,80 @@ func (in *Image) DeepCopy() *Image { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfig) DeepCopyInto(out *KubeletConfig) { + *out = *in + if in.CPUManagerPolicy != nil { + in, out := &in.CPUManagerPolicy, &out.CPUManagerPolicy + *out = new(string) + **out = **in + } + if in.CPUCfsQuota != nil { + in, out := &in.CPUCfsQuota, &out.CPUCfsQuota + *out = new(bool) + **out = **in + } + if in.CPUCfsQuotaPeriod != nil { + in, out := &in.CPUCfsQuotaPeriod, &out.CPUCfsQuotaPeriod + *out = new(string) + **out = **in + } + if in.ImageGcHighThreshold != nil { + in, out := &in.ImageGcHighThreshold, &out.ImageGcHighThreshold + *out = new(int32) + **out = **in + } + if in.ImageGcLowThreshold != nil { + in, out := &in.ImageGcLowThreshold, &out.ImageGcLowThreshold + *out = new(int32) + **out = **in + } + if in.TopologyManagerPolicy != nil { + in, out := &in.TopologyManagerPolicy, &out.TopologyManagerPolicy + *out = new(string) + **out = **in + } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = new([]string) + if **in != nil { + in, out := *in, *out + *out = make([]string, len(*in)) + copy(*out, *in) + } + } + if in.FailSwapOn != nil { + in, out := &in.FailSwapOn, &out.FailSwapOn + *out = new(bool) + **out = **in + } + if in.ContainerLogMaxSizeMB != nil { + in, out := &in.ContainerLogMaxSizeMB, &out.ContainerLogMaxSizeMB + *out = new(int32) + **out = **in + } + if in.ContainerLogMaxFiles != nil { + in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles + *out = new(int32) + **out = **in + } + if in.PodMaxPids != nil { + in, out := &in.PodMaxPids, &out.PodMaxPids + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfig. +func (in *KubeletConfig) DeepCopy() *KubeletConfig { + if in == nil { + return nil + } + out := new(KubeletConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = *in diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 6ac1515615c..b92820ddc15 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" "github.com/go-logr/logr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -504,6 +505,28 @@ func (s *ManagedControlPlaneScope) GetAgentPoolSpecs(ctx context.Context) ([]azu ammp.Replicas = *ownerPool.Spec.Replicas } + if pool.Spec.VnetSubnetID != nil { + ammp.VnetSubnetID = *pool.Spec.VnetSubnetID + } + + if pool.Spec.KubeletConfig != nil { + ammp.KubeletConfig = (*infrav1.KubeletConfig)(pool.Spec.KubeletConfig) + } + + if pool.Spec.AutoScaling != nil { + ammp.EnableAutoScaling = to.BoolPtr(true) + ammp.MaxCount = pool.Spec.AutoScaling.MaxCount + ammp.MinCount = pool.Spec.AutoScaling.MinCount + } + ammp.EnableFIPS = pool.Spec.EnableFIPS + ammp.EnableNodePublicIP = pool.Spec.EnableNodePublicIP + ammp.AvailabilityZones = pool.Spec.AvailabilityZones + ammp.NodeLabels = pool.Spec.NodeLabels + ammp.NodeTaints = pool.Spec.NodeTaints + ammp.OsDiskType = pool.Spec.OsDiskType + ammp.ScaleSetPriority = pool.Spec.ScaleSetPriority + ammp.MaxPods = pool.Spec.MaxPods + ammps = append(ammps, ammp) } @@ -547,6 +570,28 @@ func (s *ManagedControlPlaneScope) AgentPoolSpec() azure.AgentPoolSpec { agentPoolSpec.OSDiskSizeGB = *s.InfraMachinePool.Spec.OSDiskSizeGB } + if s.InfraMachinePool.Spec.VnetSubnetID != nil { + agentPoolSpec.VnetSubnetID = *s.InfraMachinePool.Spec.VnetSubnetID + } + + if s.InfraMachinePool.Spec.KubeletConfig != nil { + agentPoolSpec.KubeletConfig = (*infrav1.KubeletConfig)(s.InfraMachinePool.Spec.KubeletConfig) + } + + if s.InfraMachinePool.Spec.AutoScaling != nil { + agentPoolSpec.EnableAutoScaling = to.BoolPtr(true) + agentPoolSpec.MaxCount = s.InfraMachinePool.Spec.AutoScaling.MaxCount + agentPoolSpec.MinCount = s.InfraMachinePool.Spec.AutoScaling.MinCount + } + agentPoolSpec.EnableFIPS = s.InfraMachinePool.Spec.EnableFIPS + agentPoolSpec.EnableNodePublicIP = s.InfraMachinePool.Spec.EnableNodePublicIP + agentPoolSpec.NodeLabels = s.InfraMachinePool.Spec.NodeLabels + agentPoolSpec.NodeTaints = s.InfraMachinePool.Spec.NodeTaints + agentPoolSpec.OsDiskType = s.InfraMachinePool.Spec.OsDiskType + agentPoolSpec.AvailabilityZones = s.InfraMachinePool.Spec.AvailabilityZones + agentPoolSpec.ScaleSetPriority = s.InfraMachinePool.Spec.ScaleSetPriority + agentPoolSpec.MaxPods = s.InfraMachinePool.Spec.MaxPods + return agentPoolSpec } diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 80373bdbc5b..9516a2272bf 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -80,6 +80,28 @@ func (s *Service) Reconcile(ctx context.Context) error { }, } + if agentPoolSpec.EnableAutoScaling != nil { + profile.EnableAutoScaling = agentPoolSpec.EnableAutoScaling + profile.MaxCount = agentPoolSpec.MaxCount + profile.MinCount = agentPoolSpec.MinCount + } + + profile.EnableFIPS = agentPoolSpec.EnableFIPS + profile.EnableNodePublicIP = agentPoolSpec.EnableNodePublicIP + profile.NodeLabels = agentPoolSpec.NodeLabels + profile.NodeTaints = &agentPoolSpec.NodeTaints + profile.AvailabilityZones = &agentPoolSpec.AvailabilityZones + profile.MaxPods = agentPoolSpec.MaxPods + if agentPoolSpec.OsDiskType != nil { + profile.OsDiskType = containerservice.OSDiskType(*agentPoolSpec.OsDiskType) + } + if agentPoolSpec.ScaleSetPriority != nil { + profile.ScaleSetPriority = containerservice.ScaleSetPriority(*agentPoolSpec.ScaleSetPriority) + } + if agentPoolSpec.KubeletConfig != nil { + profile.KubeletConfig = (*containerservice.KubeletConfig)(agentPoolSpec.KubeletConfig) + } + existingPool, err := s.Client.Get(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name) if err != nil && !azure.ResourceNotFound(err) { return errors.Wrap(err, "failed to get existing agent pool") @@ -109,6 +131,10 @@ func (s *Service) Reconcile(ctx context.Context) error { Count: existingPool.Count, OrchestratorVersion: existingPool.OrchestratorVersion, Mode: existingPool.Mode, + MaxCount: existingPool.MaxCount, + MinCount: existingPool.MinCount, + EnableAutoScaling: existingPool.EnableAutoScaling, + NodeLabels: existingPool.NodeLabels, }, } @@ -117,6 +143,10 @@ func (s *Service) Reconcile(ctx context.Context) error { Count: profile.Count, OrchestratorVersion: profile.OrchestratorVersion, Mode: profile.Mode, + MaxCount: profile.MaxCount, + MinCount: profile.MinCount, + EnableAutoScaling: profile.EnableAutoScaling, + NodeLabels: profile.NodeLabels, }, } diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index eb0a70d8271..d974d947579 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -145,6 +145,31 @@ func New(scope ManagedClusterScope) *Service { } } +// SetAgentPool set the ManagedClusterAgentPoolProfile values. +func SetAgentPool(profile *containerservice.ManagedClusterAgentPoolProfile, pool *azure.AgentPoolSpec) { + if pool.OsDiskType != nil { + profile.OsDiskType = containerservice.OSDiskType(*pool.OsDiskType) + } + + if pool.ScaleSetPriority != nil { + profile.ScaleSetPriority = containerservice.ScaleSetPriority(*pool.ScaleSetPriority) + } + + if pool.KubeletConfig != nil { + profile.KubeletConfig = (*containerservice.KubeletConfig)(pool.KubeletConfig) + } + + profile.MaxCount = pool.MaxCount + profile.MinCount = pool.MinCount + profile.EnableAutoScaling = pool.EnableAutoScaling + profile.EnableFIPS = pool.EnableFIPS + profile.EnableNodePublicIP = pool.EnableNodePublicIP + profile.NodeLabels = pool.NodeLabels + profile.NodeTaints = &pool.NodeTaints + profile.AvailabilityZones = &pool.AvailabilityZones + profile.MaxPods = pool.MaxPods +} + // Reconcile idempotently creates or updates a managed cluster, if possible. func (s *Service) Reconcile(ctx context.Context) error { ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.Service.Reconcile") @@ -239,9 +264,15 @@ func (s *Service) Reconcile(ctx context.Context) error { OsDiskSizeGB: &pool.OSDiskSizeGB, Count: &pool.Replicas, Type: containerservice.AgentPoolTypeVirtualMachineScaleSets, - VnetSubnetID: &managedClusterSpec.VnetSubnetID, Mode: containerservice.AgentPoolMode(pool.Mode), } + if pool.VnetSubnetID != "" { + profile.VnetSubnetID = &pool.VnetSubnetID + } else { + profile.VnetSubnetID = &managedClusterSpec.VnetSubnetID + } + + SetAgentPool(&profile, &pool) *managedCluster.AgentPoolProfiles = append(*managedCluster.AgentPoolProfiles, profile) } diff --git a/azure/types.go b/azure/types.go index 0cf5e361969..1ae120deece 100644 --- a/azure/types.go +++ b/azure/types.go @@ -434,4 +434,40 @@ type AgentPoolSpec struct { // Mode represents mode of an agent pool. Possible values include: 'System', 'User'. Mode string + + // Max count for auto scaling + MaxCount *int32 `json:"maxCount,omitempty"` + + // Min count for auto scaling + MinCount *int32 `json:"minCount,omitempty"` + + // Enable auto scaling + EnableAutoScaling *bool `json:"EnableAutoScaling,omitempty"` + + // Enable FIPS node image + EnableFIPS *bool `json:"EnableFIPS,omitempty"` + + // Enable node public IP + EnableNodePublicIP *bool `json:"EnableNodePublicIP,omitempty"` + + // Node labels + NodeLabels map[string]*string `json:"NodeLabels,omitempty"` + + // Node taints + NodeTaints []string `json:"NodeTaints,omitempty"` + + // Node OS disk type + OsDiskType *string `json:"OsDiskType,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular' + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // MaxPods - Maximum number of pods that can run on a node. + MaxPods *int32 `json:"maxPods,omitempty"` + + // KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes. + KubeletConfig *infrav1.KubeletConfig `json:"kubeletConfig,omitempty"` } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index 0c226170b3d..d5c27bbcf20 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -117,6 +117,104 @@ spec: description: AzureManagedMachinePoolSpec defines the desired state of AzureManagedMachinePool. properties: + autoScaling: + description: EnableAutoScaling for AKS + properties: + maxCount: + description: MaxCount - Maximum number of nodes for auto-scaling + format: int32 + type: integer + minCount: + description: MinCount - Minimum number of nodes for auto-scaling + format: int32 + type: integer + required: + - maxCount + - minCount + type: object + availabilityZones: + description: AvailabilityZones - Availability zones for nodes. Must + use VirtualMachineScaleSets AgentPoolType. + items: + type: string + type: array + enableFIPS: + description: EnableFIPS - Whether to use FIPS enabled OS + type: boolean + enableNodePublicIP: + description: EnableNodePublicIP - Enable public IP for nodes + type: boolean + kubeletConfig: + description: KubeletConfig - KubeletConfig specifies the configuration + of kubelet on agent nodes. + properties: + allowedUnsafeSysctls: + description: AllowedUnsafeSysctls - Allowlist of unsafe sysctls + or unsafe sysctl patterns (ending in `*`). + items: + type: string + type: array + containerLogMaxFiles: + description: ContainerLogMaxFiles - The maximum number of container + log files that can be present for a container. The number must + be ≥ 2. + format: int32 + type: integer + containerLogMaxSizeMB: + description: ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) + of container log file before it is rotated. + format: int32 + type: integer + cpuCfsQuota: + description: CPUCfsQuota - Enable CPU CFS quota enforcement for + containers that specify CPU limits. + type: boolean + cpuCfsQuotaPeriod: + description: CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + type: string + cpuManagerPolicy: + description: CPUManagerPolicy - CPU Manager policy to use. + enum: + - none + - static + type: string + failSwapOn: + description: FailSwapOn - If set to true it will make the Kubelet + fail to start if swap is enabled on the node. + type: boolean + imageGcHighThreshold: + description: ImageGcHighThreshold - The percent of disk usage + after which image garbage collection is always run. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGcLowThreshold: + description: ImageGcLowThreshold - The percent of disk usage before + which image garbage collection is never run. + format: int32 + maximum: 100 + minimum: 0 + type: integer + podMaxPids: + description: PodMaxPids - The maximum number of processes per + pod. + format: int32 + type: integer + topologyManagerPolicy: + description: TopologyManagerPolicy - Topology Manager policy to + use. + enum: + - none + - best-effort + - restricted + - single-numa-node + type: string + type: object + maxPods: + description: MaxPods - Maximum number of pods that can run on a node. + format: int32 + type: integer mode: description: 'Mode - represents mode of an agent pool. Possible values include: System, User.' @@ -128,21 +226,56 @@ spec: description: Name - name of the agent pool. If not specified, CAPZ uses the name of the CR as the agent pool name. type: string + nodeLabels: + additionalProperties: + type: string + description: NodeLabels - Agent pool node labels to be persisted across + all nodes in agent pool. + type: object + nodeTaints: + description: NodeTaints - Taints added to new nodes during node pool + create and scale. For example, key=value:NoSchedule. + items: + type: string + type: array osDiskSizeGB: description: OSDiskSizeGB is the disk size for every machine in this agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. format: int32 type: integer + osDiskType: + description: 'OsDiskType - OS disk type to be used for machines in + a given agent pool. Allowed values are ''Ephemeral'' and ''Managed''. + If unspecified, defaults to ''Ephemeral'' when the VM supports ephemeral + OS and has a cache disk larger than the requested OSDiskSizeGB. + Otherwise, defaults to ''Managed''. May not be changed after creation. + Possible values include: ''Managed'', ''Ephemeral''' + enum: + - Managed + - Ephemeral + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. items: type: string type: array + scaleSetPriority: + description: 'ScaleSetPriority - ScaleSetPriority to be used to specify + virtual machine scale set priority. Default to regular. Possible + values include: ''Spot'', ''Regular''' + enum: + - Regular + - Spot + type: string sku: description: SKU is the size of the VMs in the node pool. type: string + vnetSubnetID: + description: VnetSubnetID - VNet SubnetID specifies the VNet's subnet + identifier for nodes and maybe pods + type: string required: - mode - sku @@ -196,6 +329,104 @@ spec: description: AzureManagedMachinePoolSpec defines the desired state of AzureManagedMachinePool. properties: + autoScaling: + description: EnableAutoScaling for AKS + properties: + maxCount: + description: MaxCount - Maximum number of nodes for auto-scaling + format: int32 + type: integer + minCount: + description: MinCount - Minimum number of nodes for auto-scaling + format: int32 + type: integer + required: + - maxCount + - minCount + type: object + availabilityZones: + description: AvailabilityZones - Availability zones for nodes. Must + use VirtualMachineScaleSets AgentPoolType. + items: + type: string + type: array + enableFIPS: + description: EnableFIPS - Whether to use FIPS enabled OS + type: boolean + enableNodePublicIP: + description: EnableNodePublicIP - Enable public IP for nodes + type: boolean + kubeletConfig: + description: KubeletConfig - KubeletConfig specifies the configuration + of kubelet on agent nodes. + properties: + allowedUnsafeSysctls: + description: AllowedUnsafeSysctls - Allowlist of unsafe sysctls + or unsafe sysctl patterns (ending in `*`). + items: + type: string + type: array + containerLogMaxFiles: + description: ContainerLogMaxFiles - The maximum number of container + log files that can be present for a container. The number must + be ≥ 2. + format: int32 + type: integer + containerLogMaxSizeMB: + description: ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) + of container log file before it is rotated. + format: int32 + type: integer + cpuCfsQuota: + description: CPUCfsQuota - Enable CPU CFS quota enforcement for + containers that specify CPU limits. + type: boolean + cpuCfsQuotaPeriod: + description: CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + type: string + cpuManagerPolicy: + description: CPUManagerPolicy - CPU Manager policy to use. + enum: + - none + - static + type: string + failSwapOn: + description: FailSwapOn - If set to true it will make the Kubelet + fail to start if swap is enabled on the node. + type: boolean + imageGcHighThreshold: + description: ImageGcHighThreshold - The percent of disk usage + after which image garbage collection is always run. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGcLowThreshold: + description: ImageGcLowThreshold - The percent of disk usage before + which image garbage collection is never run. + format: int32 + maximum: 100 + minimum: 0 + type: integer + podMaxPids: + description: PodMaxPids - The maximum number of processes per + pod. + format: int32 + type: integer + topologyManagerPolicy: + description: TopologyManagerPolicy - Topology Manager policy to + use. + enum: + - none + - best-effort + - restricted + - single-numa-node + type: string + type: object + maxPods: + description: MaxPods - Maximum number of pods that can run on a node. + format: int32 + type: integer mode: description: 'Mode - represents mode of an agent pool. Possible values include: System, User.' @@ -207,21 +438,56 @@ spec: description: Name - name of the agent pool. If not specified, CAPZ uses the name of the CR as the agent pool name. type: string + nodeLabels: + additionalProperties: + type: string + description: NodeLabels - Agent pool node labels to be persisted across + all nodes in agent pool. + type: object + nodeTaints: + description: NodeTaints - Taints added to new nodes during node pool + create and scale. For example, key=value:NoSchedule. + items: + type: string + type: array osDiskSizeGB: description: OSDiskSizeGB is the disk size for every machine in this agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. format: int32 type: integer + osDiskType: + description: 'OsDiskType - OS disk type to be used for machines in + a given agent pool. Allowed values are ''Ephemeral'' and ''Managed''. + If unspecified, defaults to ''Ephemeral'' when the VM supports ephemeral + OS and has a cache disk larger than the requested OSDiskSizeGB. + Otherwise, defaults to ''Managed''. May not be changed after creation. + Possible values include: ''Managed'', ''Ephemeral''' + enum: + - Managed + - Ephemeral + type: string providerIDList: description: ProviderIDList is the unique identifier as specified by the cloud provider. items: type: string type: array + scaleSetPriority: + description: 'ScaleSetPriority - ScaleSetPriority to be used to specify + virtual machine scale set priority. Default to regular. Possible + values include: ''Spot'', ''Regular''' + enum: + - Regular + - Spot + type: string sku: description: SKU is the size of the VMs in the node pool. type: string + vnetSubnetID: + description: VnetSubnetID - VNet SubnetID specifies the VNet's subnet + identifier for nodes and maybe pods + type: string required: - mode - sku diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go index b02f694cd07..691d3bde5b6 100644 --- a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go @@ -36,8 +36,100 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { // if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok { return err } + + //if restored.ObjectMeta.Annotations != nil { + // if dst.ObjectMeta.Annotations == nil { + // dst.ObjectMeta.Annotations = expv1beta1.AzureManagedMachinePool{}.ObjectMeta.Annotations + // } + //} else { + // dst.ObjectMeta.Annotations = restored.ObjectMeta.Annotations + //} dst.Spec.Name = restored.Spec.Name + dst.Spec.AutoScaling = restored.Spec.AutoScaling + dst.Spec.EnableFIPS = restored.Spec.EnableFIPS + dst.Spec.EnableNodePublicIP = restored.Spec.EnableNodePublicIP + dst.Spec.OsDiskType = restored.Spec.OsDiskType + dst.Spec.MaxPods = restored.Spec.MaxPods + dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority + dst.Spec.VnetSubnetID = restored.Spec.VnetSubnetID + dst.Spec.KubeletConfig = restored.Spec.KubeletConfig + //dst.Spec.KubeletConfig.AllowedUnsafeSysctls = &[]string{} + //dst.Spec.NodeTaints = []string{} + //dst.Spec.AvailabilityZones = []string{} + + if restored.Spec.NodeLabels != nil { + if len(restored.Spec.NodeLabels) == 0 { + dst.Spec.NodeLabels = map[string]*string{} + } else { + dst.Spec.NodeLabels = restored.Spec.NodeLabels + } + //for key, value := range restored.Spec.NodeLabels { + // dst.Spec.NodeLabels[key] = value + //} + } else { + dst.Spec.NodeLabels = nil + } + + // + // if len(dst.Spec.AvailabilityZones) == 0 { + // restore nil value if nothing has changed since conversion + // dst.Spec.AvailabilityZones = nil + //} + //} else { + // + //} + + //if restored.Spec.AvailabilityZones == nil { + // dst.Spec.AvailabilityZones = nil + //} else { + // if len(restored.Spec.AvailabilityZones) == 0 { + // dst.Spec.AvailabilityZones = []string{} + // } else { + // dst.Spec.AvailabilityZones = restored.Spec.AvailabilityZones + // } + //} + + dst.Spec.AvailabilityZones = restored.Spec.AvailabilityZones + if len(dst.Spec.AvailabilityZones) == 0 { + dst.Spec.AvailabilityZones = nil + } + + dst.Spec.NodeTaints = restored.Spec.NodeTaints + if len(dst.Spec.NodeTaints) == 0 { + dst.Spec.NodeTaints = nil + } + + //if restored.Spec.NodeTaints == nil { + // dst.Spec.NodeTaints = nil + //} else { + // if len(restored.Spec.NodeTaints) == 0 { + // dst.Spec.NodeTaints = []string{} + // } else { + // dst.Spec.NodeTaints = restored.Spec.NodeTaints + // } + //} + + dst.Spec.KubeletConfig = restored.Spec.KubeletConfig + //if restored.Spec.KubeletConfig.AllowedUnsafeSysctls == nil { + // dst.Spec.KubeletConfig.AllowedUnsafeSysctls = nil + //} + + //if restored.Spec.KubeletConfig != nil { + // dst.Spec.KubeletConfig = restored.Spec.KubeletConfig + //} else { + // dst.Spec.KubeletConfig = nil + //} + + //if restored.Spec.KubeletConfig.AllowedUnsafeSysctls != nil { + // dst.Spec.KubeletConfig.AllowedUnsafeSysctls = restored.Spec.KubeletConfig.AllowedUnsafeSysctls + //} else { + // dst.Spec.KubeletConfig.AllowedUnsafeSysctls = nil + //} + + if len(dst.Annotations) == 0 { + dst.Annotations = nil + } return nil } diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 9905e89f7cb..78dfceaf80d 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -879,6 +879,17 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha3_AzureManagedMac out.SKU = in.SKU out.OSDiskSizeGB = (*int32)(unsafe.Pointer(in.OSDiskSizeGB)) out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) + // WARNING: in.AutoScaling requires manual conversion: does not exist in peer-type + // WARNING: in.EnableNodePublicIP requires manual conversion: does not exist in peer-type + // WARNING: in.EnableFIPS requires manual conversion: does not exist in peer-type + // WARNING: in.OsDiskType requires manual conversion: does not exist in peer-type + // WARNING: in.NodeLabels requires manual conversion: does not exist in peer-type + // WARNING: in.NodeTaints requires manual conversion: does not exist in peer-type + // WARNING: in.VnetSubnetID requires manual conversion: does not exist in peer-type + // WARNING: in.AvailabilityZones requires manual conversion: does not exist in peer-type + // WARNING: in.ScaleSetPriority requires manual conversion: does not exist in peer-type + // WARNING: in.MaxPods requires manual conversion: does not exist in peer-type + // WARNING: in.KubeletConfig requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_types.go b/exp/api/v1alpha4/azuremanagedmachinepool_types.go index 646c8a3b859..2a78bfaa4bd 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_types.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_types.go @@ -56,6 +56,116 @@ type AzureManagedMachinePoolSpec struct { // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` + + // EnableAutoScaling for AKS + // +optional + AutoScaling *AutoScaling `json:"autoScaling,omitempty"` + + // EnableNodePublicIP - Enable public IP for nodes + // +optional + EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + + // EnableFIPS - Whether to use FIPS enabled OS + // +optional + EnableFIPS *bool `json:"enableFIPS,omitempty"` + + // OsDiskType - OS disk type to be used for machines in a given agent pool. Allowed values are 'Ephemeral' and 'Managed'. If unspecified, defaults to 'Ephemeral' when the VM supports ephemeral OS and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to 'Managed'. May not be changed after creation. Possible values include: 'Managed', 'Ephemeral' + // +kubebuilder:validation:Enum=Managed;Ephemeral + // +optional + OsDiskType *string `json:"osDiskType,omitempty"` + + // NodeLabels - Agent pool node labels to be persisted across all nodes in agent pool. + // +optional + NodeLabels map[string]*string `json:"nodeLabels,omitempty"` + + // NodeTaints - Taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. + // +optional + NodeTaints []string `json:"nodeTaints,omitempty"` + + // VnetSubnetID - VNet SubnetID specifies the VNet's subnet identifier for nodes and maybe pods + // +optional + VnetSubnetID *string `json:"vnetSubnetID,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + // +optional + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular' + // +kubebuilder:validation:Enum=Regular;Spot + // +optional + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // MaxPods - Maximum number of pods that can run on a node. + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + + // KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes. + // +optional + KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` +} + +// AutoScaling config. +type AutoScaling struct { + // MaxCount - Maximum number of nodes for auto-scaling + // +kubebuilder:validation:Required + MaxCount *int32 `json:"maxCount"` + + // MinCount - Minimum number of nodes for auto-scaling + // +kubebuilder:validation:Required + MinCount *int32 `json:"minCount"` +} + +// KubeletConfig kubelet configurations of agent nodes. +type KubeletConfig struct { + // CPUManagerPolicy - CPU Manager policy to use. + // +kubebuilder:validation:Enum=none;static + // +optional + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + + // CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits. + // +optional + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + + // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + // +optional + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + + // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + // +optional + // +kubebuilder:validation:Maximum=100 + // +kubebuilder:validation:Minimum=0 + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + + // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + // +optional + // +kubebuilder:validation:Maximum=100 + // +kubebuilder:validation:Minimum=0 + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + + // TopologyManagerPolicy - Topology Manager policy to use. + // +kubebuilder:validation:Enum=none;best-effort;restricted;single-numa-node + // +optional + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` + + // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + // +optional + AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"` + + // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. + // +optional + FailSwapOn *bool `json:"failSwapOn,omitempty"` + + // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + // +optional + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + + // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. + // +optional + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + + // PodMaxPids - The maximum number of processes per pod. + // +optional + PodMaxPids *int32 `json:"podMaxPids,omitempty"` } // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool. diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index bf819586e87..b518650104d 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -63,6 +63,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*AutoScaling)(nil), (*v1beta1.AutoScaling)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_AutoScaling_To_v1beta1_AutoScaling(a.(*AutoScaling), b.(*v1beta1.AutoScaling), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta1.AutoScaling)(nil), (*AutoScaling)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AutoScaling_To_v1alpha4_AutoScaling(a.(*v1beta1.AutoScaling), b.(*AutoScaling), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*AzureMachinePool)(nil), (*v1beta1.AzureMachinePool)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_AzureMachinePool_To_v1beta1_AzureMachinePool(a.(*AzureMachinePool), b.(*v1beta1.AzureMachinePool), scope) }); err != nil { @@ -293,6 +303,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KubeletConfig)(nil), (*v1beta1.KubeletConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_KubeletConfig_To_v1beta1_KubeletConfig(a.(*KubeletConfig), b.(*v1beta1.KubeletConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta1.KubeletConfig)(nil), (*KubeletConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_KubeletConfig_To_v1alpha4_KubeletConfig(a.(*v1beta1.KubeletConfig), b.(*KubeletConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*LoadBalancerProfile)(nil), (*v1beta1.LoadBalancerProfile)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_LoadBalancerProfile_To_v1beta1_LoadBalancerProfile(a.(*LoadBalancerProfile), b.(*v1beta1.LoadBalancerProfile), scope) }); err != nil { @@ -424,6 +444,28 @@ func Convert_v1beta1_APIServerAccessProfile_To_v1alpha4_APIServerAccessProfile(i return autoConvert_v1beta1_APIServerAccessProfile_To_v1alpha4_APIServerAccessProfile(in, out, s) } +func autoConvert_v1alpha4_AutoScaling_To_v1beta1_AutoScaling(in *AutoScaling, out *v1beta1.AutoScaling, s conversion.Scope) error { + out.MaxCount = (*int32)(unsafe.Pointer(in.MaxCount)) + out.MinCount = (*int32)(unsafe.Pointer(in.MinCount)) + return nil +} + +// Convert_v1alpha4_AutoScaling_To_v1beta1_AutoScaling is an autogenerated conversion function. +func Convert_v1alpha4_AutoScaling_To_v1beta1_AutoScaling(in *AutoScaling, out *v1beta1.AutoScaling, s conversion.Scope) error { + return autoConvert_v1alpha4_AutoScaling_To_v1beta1_AutoScaling(in, out, s) +} + +func autoConvert_v1beta1_AutoScaling_To_v1alpha4_AutoScaling(in *v1beta1.AutoScaling, out *AutoScaling, s conversion.Scope) error { + out.MaxCount = (*int32)(unsafe.Pointer(in.MaxCount)) + out.MinCount = (*int32)(unsafe.Pointer(in.MinCount)) + return nil +} + +// Convert_v1beta1_AutoScaling_To_v1alpha4_AutoScaling is an autogenerated conversion function. +func Convert_v1beta1_AutoScaling_To_v1alpha4_AutoScaling(in *v1beta1.AutoScaling, out *AutoScaling, s conversion.Scope) error { + return autoConvert_v1beta1_AutoScaling_To_v1alpha4_AutoScaling(in, out, s) +} + func autoConvert_v1alpha4_AzureMachinePool_To_v1beta1_AzureMachinePool(in *AzureMachinePool, out *v1beta1.AzureMachinePool, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha4_AzureMachinePoolSpec_To_v1beta1_AzureMachinePoolSpec(&in.Spec, &out.Spec, s); err != nil { @@ -1160,6 +1202,17 @@ func autoConvert_v1alpha4_AzureManagedMachinePoolSpec_To_v1beta1_AzureManagedMac out.SKU = in.SKU out.OSDiskSizeGB = (*int32)(unsafe.Pointer(in.OSDiskSizeGB)) out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) + out.AutoScaling = (*v1beta1.AutoScaling)(unsafe.Pointer(in.AutoScaling)) + out.EnableNodePublicIP = (*bool)(unsafe.Pointer(in.EnableNodePublicIP)) + out.EnableFIPS = (*bool)(unsafe.Pointer(in.EnableFIPS)) + out.OsDiskType = (*string)(unsafe.Pointer(in.OsDiskType)) + out.NodeLabels = *(*map[string]*string)(unsafe.Pointer(&in.NodeLabels)) + out.NodeTaints = *(*[]string)(unsafe.Pointer(&in.NodeTaints)) + out.VnetSubnetID = (*string)(unsafe.Pointer(in.VnetSubnetID)) + out.AvailabilityZones = *(*[]string)(unsafe.Pointer(&in.AvailabilityZones)) + out.ScaleSetPriority = (*string)(unsafe.Pointer(in.ScaleSetPriority)) + out.MaxPods = (*int32)(unsafe.Pointer(in.MaxPods)) + out.KubeletConfig = (*v1beta1.KubeletConfig)(unsafe.Pointer(in.KubeletConfig)) return nil } @@ -1174,6 +1227,17 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac out.SKU = in.SKU out.OSDiskSizeGB = (*int32)(unsafe.Pointer(in.OSDiskSizeGB)) out.ProviderIDList = *(*[]string)(unsafe.Pointer(&in.ProviderIDList)) + out.AutoScaling = (*AutoScaling)(unsafe.Pointer(in.AutoScaling)) + out.EnableNodePublicIP = (*bool)(unsafe.Pointer(in.EnableNodePublicIP)) + out.EnableFIPS = (*bool)(unsafe.Pointer(in.EnableFIPS)) + out.OsDiskType = (*string)(unsafe.Pointer(in.OsDiskType)) + out.NodeLabels = *(*map[string]*string)(unsafe.Pointer(&in.NodeLabels)) + out.NodeTaints = *(*[]string)(unsafe.Pointer(&in.NodeTaints)) + out.VnetSubnetID = (*string)(unsafe.Pointer(in.VnetSubnetID)) + out.AvailabilityZones = *(*[]string)(unsafe.Pointer(&in.AvailabilityZones)) + out.ScaleSetPriority = (*string)(unsafe.Pointer(in.ScaleSetPriority)) + out.MaxPods = (*int32)(unsafe.Pointer(in.MaxPods)) + out.KubeletConfig = (*KubeletConfig)(unsafe.Pointer(in.KubeletConfig)) return nil } @@ -1208,6 +1272,46 @@ func Convert_v1beta1_AzureManagedMachinePoolStatus_To_v1alpha4_AzureManagedMachi return autoConvert_v1beta1_AzureManagedMachinePoolStatus_To_v1alpha4_AzureManagedMachinePoolStatus(in, out, s) } +func autoConvert_v1alpha4_KubeletConfig_To_v1beta1_KubeletConfig(in *KubeletConfig, out *v1beta1.KubeletConfig, s conversion.Scope) error { + out.CPUManagerPolicy = (*string)(unsafe.Pointer(in.CPUManagerPolicy)) + out.CPUCfsQuota = (*bool)(unsafe.Pointer(in.CPUCfsQuota)) + out.CPUCfsQuotaPeriod = (*string)(unsafe.Pointer(in.CPUCfsQuotaPeriod)) + out.ImageGcHighThreshold = (*int32)(unsafe.Pointer(in.ImageGcHighThreshold)) + out.ImageGcLowThreshold = (*int32)(unsafe.Pointer(in.ImageGcLowThreshold)) + out.TopologyManagerPolicy = (*string)(unsafe.Pointer(in.TopologyManagerPolicy)) + out.AllowedUnsafeSysctls = (*[]string)(unsafe.Pointer(in.AllowedUnsafeSysctls)) + out.FailSwapOn = (*bool)(unsafe.Pointer(in.FailSwapOn)) + out.ContainerLogMaxSizeMB = (*int32)(unsafe.Pointer(in.ContainerLogMaxSizeMB)) + out.ContainerLogMaxFiles = (*int32)(unsafe.Pointer(in.ContainerLogMaxFiles)) + out.PodMaxPids = (*int32)(unsafe.Pointer(in.PodMaxPids)) + return nil +} + +// Convert_v1alpha4_KubeletConfig_To_v1beta1_KubeletConfig is an autogenerated conversion function. +func Convert_v1alpha4_KubeletConfig_To_v1beta1_KubeletConfig(in *KubeletConfig, out *v1beta1.KubeletConfig, s conversion.Scope) error { + return autoConvert_v1alpha4_KubeletConfig_To_v1beta1_KubeletConfig(in, out, s) +} + +func autoConvert_v1beta1_KubeletConfig_To_v1alpha4_KubeletConfig(in *v1beta1.KubeletConfig, out *KubeletConfig, s conversion.Scope) error { + out.CPUManagerPolicy = (*string)(unsafe.Pointer(in.CPUManagerPolicy)) + out.CPUCfsQuota = (*bool)(unsafe.Pointer(in.CPUCfsQuota)) + out.CPUCfsQuotaPeriod = (*string)(unsafe.Pointer(in.CPUCfsQuotaPeriod)) + out.ImageGcHighThreshold = (*int32)(unsafe.Pointer(in.ImageGcHighThreshold)) + out.ImageGcLowThreshold = (*int32)(unsafe.Pointer(in.ImageGcLowThreshold)) + out.TopologyManagerPolicy = (*string)(unsafe.Pointer(in.TopologyManagerPolicy)) + out.AllowedUnsafeSysctls = (*[]string)(unsafe.Pointer(in.AllowedUnsafeSysctls)) + out.FailSwapOn = (*bool)(unsafe.Pointer(in.FailSwapOn)) + out.ContainerLogMaxSizeMB = (*int32)(unsafe.Pointer(in.ContainerLogMaxSizeMB)) + out.ContainerLogMaxFiles = (*int32)(unsafe.Pointer(in.ContainerLogMaxFiles)) + out.PodMaxPids = (*int32)(unsafe.Pointer(in.PodMaxPids)) + return nil +} + +// Convert_v1beta1_KubeletConfig_To_v1alpha4_KubeletConfig is an autogenerated conversion function. +func Convert_v1beta1_KubeletConfig_To_v1alpha4_KubeletConfig(in *v1beta1.KubeletConfig, out *KubeletConfig, s conversion.Scope) error { + return autoConvert_v1beta1_KubeletConfig_To_v1alpha4_KubeletConfig(in, out, s) +} + func autoConvert_v1alpha4_LoadBalancerProfile_To_v1beta1_LoadBalancerProfile(in *LoadBalancerProfile, out *v1beta1.LoadBalancerProfile, s conversion.Scope) error { out.ManagedOutboundIPs = (*int32)(unsafe.Pointer(in.ManagedOutboundIPs)) out.OutboundIPPrefixes = *(*[]string)(unsafe.Pointer(&in.OutboundIPPrefixes)) diff --git a/exp/api/v1alpha4/zz_generated.deepcopy.go b/exp/api/v1alpha4/zz_generated.deepcopy.go index 7b264879ed2..90f7013e6e7 100644 --- a/exp/api/v1alpha4/zz_generated.deepcopy.go +++ b/exp/api/v1alpha4/zz_generated.deepcopy.go @@ -85,6 +85,31 @@ func (in *APIServerAccessProfile) DeepCopy() *APIServerAccessProfile { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoScaling) DeepCopyInto(out *AutoScaling) { + *out = *in + if in.MaxCount != nil { + in, out := &in.MaxCount, &out.MaxCount + *out = new(int32) + **out = **in + } + if in.MinCount != nil { + in, out := &in.MinCount, &out.MinCount + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoScaling. +func (in *AutoScaling) DeepCopy() *AutoScaling { + if in == nil { + return nil + } + out := new(AutoScaling) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureMachinePool) DeepCopyInto(out *AzureMachinePool) { *out = *in @@ -765,6 +790,71 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = make([]string, len(*in)) copy(*out, *in) } + if in.AutoScaling != nil { + in, out := &in.AutoScaling, &out.AutoScaling + *out = new(AutoScaling) + (*in).DeepCopyInto(*out) + } + if in.EnableNodePublicIP != nil { + in, out := &in.EnableNodePublicIP, &out.EnableNodePublicIP + *out = new(bool) + **out = **in + } + if in.EnableFIPS != nil { + in, out := &in.EnableFIPS, &out.EnableFIPS + *out = new(bool) + **out = **in + } + if in.OsDiskType != nil { + in, out := &in.OsDiskType, &out.OsDiskType + *out = new(string) + **out = **in + } + if in.NodeLabels != nil { + in, out := &in.NodeLabels, &out.NodeLabels + *out = make(map[string]*string, len(*in)) + for key, val := range *in { + var outVal *string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(string) + **out = **in + } + (*out)[key] = outVal + } + } + if in.NodeTaints != nil { + in, out := &in.NodeTaints, &out.NodeTaints + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.VnetSubnetID != nil { + in, out := &in.VnetSubnetID, &out.VnetSubnetID + *out = new(string) + **out = **in + } + if in.AvailabilityZones != nil { + in, out := &in.AvailabilityZones, &out.AvailabilityZones + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ScaleSetPriority != nil { + in, out := &in.ScaleSetPriority, &out.ScaleSetPriority + *out = new(string) + **out = **in + } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } + if in.KubeletConfig != nil { + in, out := &in.KubeletConfig, &out.KubeletConfig + *out = new(KubeletConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolSpec. @@ -802,6 +892,80 @@ func (in *AzureManagedMachinePoolStatus) DeepCopy() *AzureManagedMachinePoolStat return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfig) DeepCopyInto(out *KubeletConfig) { + *out = *in + if in.CPUManagerPolicy != nil { + in, out := &in.CPUManagerPolicy, &out.CPUManagerPolicy + *out = new(string) + **out = **in + } + if in.CPUCfsQuota != nil { + in, out := &in.CPUCfsQuota, &out.CPUCfsQuota + *out = new(bool) + **out = **in + } + if in.CPUCfsQuotaPeriod != nil { + in, out := &in.CPUCfsQuotaPeriod, &out.CPUCfsQuotaPeriod + *out = new(string) + **out = **in + } + if in.ImageGcHighThreshold != nil { + in, out := &in.ImageGcHighThreshold, &out.ImageGcHighThreshold + *out = new(int32) + **out = **in + } + if in.ImageGcLowThreshold != nil { + in, out := &in.ImageGcLowThreshold, &out.ImageGcLowThreshold + *out = new(int32) + **out = **in + } + if in.TopologyManagerPolicy != nil { + in, out := &in.TopologyManagerPolicy, &out.TopologyManagerPolicy + *out = new(string) + **out = **in + } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = new([]string) + if **in != nil { + in, out := *in, *out + *out = make([]string, len(*in)) + copy(*out, *in) + } + } + if in.FailSwapOn != nil { + in, out := &in.FailSwapOn, &out.FailSwapOn + *out = new(bool) + **out = **in + } + if in.ContainerLogMaxSizeMB != nil { + in, out := &in.ContainerLogMaxSizeMB, &out.ContainerLogMaxSizeMB + *out = new(int32) + **out = **in + } + if in.ContainerLogMaxFiles != nil { + in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles + *out = new(int32) + **out = **in + } + if in.PodMaxPids != nil { + in, out := &in.PodMaxPids, &out.PodMaxPids + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfig. +func (in *KubeletConfig) DeepCopy() *KubeletConfig { + if in == nil { + return nil + } + out := new(KubeletConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerProfile) DeepCopyInto(out *LoadBalancerProfile) { *out = *in diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go index b18dc8df004..f35f68c7151 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_types.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_types.go @@ -57,6 +57,116 @@ type AzureManagedMachinePoolSpec struct { // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` + + // EnableAutoScaling for AKS + // +optional + AutoScaling *AutoScaling `json:"autoScaling,omitempty"` + + // EnableNodePublicIP - Enable public IP for nodes + // +optional + EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + + // EnableFIPS - Whether to use FIPS enabled OS + // +optional + EnableFIPS *bool `json:"enableFIPS,omitempty"` + + // OsDiskType - OS disk type to be used for machines in a given agent pool. Allowed values are 'Ephemeral' and 'Managed'. If unspecified, defaults to 'Ephemeral' when the VM supports ephemeral OS and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to 'Managed'. May not be changed after creation. Possible values include: 'Managed', 'Ephemeral' + // +kubebuilder:validation:Enum=Managed;Ephemeral + // +optional + OsDiskType *string `json:"osDiskType,omitempty"` + + // NodeLabels - Agent pool node labels to be persisted across all nodes in agent pool. + // +optional + NodeLabels map[string]*string `json:"nodeLabels,omitempty"` + + // NodeTaints - Taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. + // +optional + NodeTaints []string `json:"nodeTaints,omitempty"` + + // VnetSubnetID - VNet SubnetID specifies the VNet's subnet identifier for nodes and maybe pods + // +optional + VnetSubnetID *string `json:"vnetSubnetID,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + // +optional + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular' + // +kubebuilder:validation:Enum=Regular;Spot + // +optional + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // MaxPods - Maximum number of pods that can run on a node. + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + + // KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes. + // +optional + KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` +} + +// AutoScaling config. +type AutoScaling struct { + // MaxCount - Maximum number of nodes for auto-scaling + // +kubebuilder:validation:Required + MaxCount *int32 `json:"maxCount"` + + // MinCount - Minimum number of nodes for auto-scaling + // +kubebuilder:validation:Required + MinCount *int32 `json:"minCount"` +} + +// KubeletConfig kubelet configurations of agent nodes. +type KubeletConfig struct { + // CPUManagerPolicy - CPU Manager policy to use. + // +kubebuilder:validation:Enum=none;static + // +optional + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + + // CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits. + // +optional + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + + // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + // +optional + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + + // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + // +optional + // +kubebuilder:validation:Maximum=100 + // +kubebuilder:validation:Minimum=0 + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + + // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + // +optional + // +kubebuilder:validation:Maximum=100 + // +kubebuilder:validation:Minimum=0 + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + + // TopologyManagerPolicy - Topology Manager policy to use. + // +kubebuilder:validation:Enum=none;best-effort;restricted;single-numa-node + // +optional + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` + + // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + // +optional + AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"` + + // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. + // +optional + FailSwapOn *bool `json:"failSwapOn,omitempty"` + + // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + // +optional + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + + // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. + // +optional + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + + // PodMaxPids - The maximum number of processes per pod. + // +optional + PodMaxPids *int32 `json:"podMaxPids,omitempty"` } // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool. diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go index 922710d26f5..f9bdbd3b03b 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -18,6 +18,8 @@ package v1beta1 import ( "context" + "reflect" + "regexp" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -55,41 +57,252 @@ func (r *AzureManagedMachinePool) Default(client client.Client) { // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. func (r *AzureManagedMachinePool) ValidateCreate(client client.Client) error { azuremanagedmachinepoollog.Info("validate create", "name", r.Name) + var allErrs field.ErrorList + + // If AutoScaling is enabled, both MinCount and MaxCount should be set. + if r.Spec.AutoScaling != nil && (*r.Spec.AutoScaling.MaxCount < *r.Spec.AutoScaling.MinCount) { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "AutoScaling"), + r.Spec.AutoScaling, + "MaxCount must be greater or equal to MinCount when AutoScaling is enabled")) + } + + // NodeTaints should follow pattern key=value:effect. + if r.Spec.NodeTaints != nil { + re := regexp.MustCompile(`^.+=.+:.+$`) + for _, v := range r.Spec.NodeTaints { + if re.FindString(v) == "" { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "NodeTaints"), + r.Spec.NodeTaints, + "Node taints should follow the pattern such as key=value:effect")) + } + } + } + + // AllowedUnsafeSysctls should be one of "kernel.shm*", "kernel.msg*", "kernel.sem", "fs.mqueue.*", "net.*". + if r.Spec.KubeletConfig != nil && len(*r.Spec.KubeletConfig.AllowedUnsafeSysctls) > 0 { + for _, v := range *r.Spec.KubeletConfig.AllowedUnsafeSysctls { + switch v { + case "kernel.shm*", "kernel.msg*", "kernel.sem", "fs.mqueue.*", "net.*": + continue + } + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "KubeletConfig"), + r.Spec.KubeletConfig, + "AllowedUnsafeSysctls are \"kernel.shm*\", \"kernel.msg*\", \"kernel.sem\", \"fs.mqueue.*\", \"net.*\"")) + } + } + + if len(allErrs) != 0 { + return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePool").GroupKind(), r.Name, allErrs) + } return nil } +// ValidateUpdateHelper is a helper function to reduce cyclomatic complexity. +func ValidateUpdateHelper(old AzureManagedMachinePoolSpec, r AzureManagedMachinePoolSpec) field.ErrorList { + var updateErrs field.ErrorList + if r.SKU != old.SKU { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "SKU"), + r.SKU, + "field is immutable")) + } + + if old.OSDiskSizeGB != nil { + // Prevent OSDiskSizeGB modification if it was already set to some value + if r.OSDiskSizeGB == nil { + // unsetting the field is not allowed + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "OSDiskSizeGB"), + r.OSDiskSizeGB, + "field is immutable, unsetting is not allowed")) + } else if *r.OSDiskSizeGB != *old.OSDiskSizeGB { + // changing the field is not allowed + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "OSDiskSizeGB"), + *r.OSDiskSizeGB, + "field is immutable")) + } + } + if !reflect.DeepEqual(r.NodeTaints, old.NodeTaints) { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "NodeTaints"), + r.NodeTaints, + "field is immutable")) + } + + if old.VnetSubnetID == nil && r.VnetSubnetID != nil { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "VnetSubnetID"), + r.VnetSubnetID, + "field is immutable, setting after creation is not allowed")) + } + + if old.VnetSubnetID != nil { + if r.VnetSubnetID == nil { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "VnetSubnetID"), + r.VnetSubnetID, + "field is immutable, unsetting is not allowed")) + } else if *r.VnetSubnetID != *old.VnetSubnetID { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "VnetSubnetID"), + r.VnetSubnetID, + "field is immutable")) + } + } + + if old.MaxPods == nil && r.MaxPods != nil { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "MaxPods"), + r.MaxPods, + "field is immutable, setting after creation is not allowed")) + } + + if old.MaxPods != nil { + if r.MaxPods == nil { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "MaxPods"), + r.MaxPods, + "field is immutable, unsetting is not allowed")) + } else if *r.MaxPods != *old.MaxPods { + updateErrs = append(updateErrs, + field.Invalid( + field.NewPath("Spec", "MaxPods"), + r.MaxPods, + "field is immutable")) + } + } + + return updateErrs +} + // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. func (r *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client client.Client) error { old := oldRaw.(*AzureManagedMachinePool) var allErrs field.ErrorList - if r.Spec.SKU != old.Spec.SKU { + var updateErrs = ValidateUpdateHelper(old.Spec, r.Spec) + allErrs = append(allErrs, updateErrs...) + + if old.Spec.EnableFIPS == nil && r.Spec.EnableFIPS != nil { allErrs = append(allErrs, field.Invalid( - field.NewPath("Spec", "SKU"), - r.Spec.SKU, - "field is immutable")) + field.NewPath("Spec", "EnableFIPS"), + r.Spec.EnableFIPS, + "field is immutable, setting after creation is not allowed")) } - if old.Spec.OSDiskSizeGB != nil { - // Prevent OSDiskSizeGB modification if it was already set to some value - if r.Spec.OSDiskSizeGB == nil { - // unsetting the field is not allowed + if old.Spec.EnableFIPS != nil { + if r.Spec.EnableFIPS == nil { allErrs = append(allErrs, field.Invalid( - field.NewPath("Spec", "OSDiskSizeGB"), - r.Spec.OSDiskSizeGB, + field.NewPath("Spec", "EnableFIPS"), + r.Spec.EnableFIPS, "field is immutable, unsetting is not allowed")) - } else if *r.Spec.OSDiskSizeGB != *old.Spec.OSDiskSizeGB { - // changing the field is not allowed + } else if *r.Spec.EnableFIPS != *old.Spec.EnableFIPS { allErrs = append(allErrs, field.Invalid( - field.NewPath("Spec", "OSDiskSizeGB"), - *r.Spec.OSDiskSizeGB, + field.NewPath("Spec", "EnableFIPS"), + r.Spec.EnableFIPS, + "field is immutable")) + } + } + + if old.Spec.EnableNodePublicIP == nil && r.Spec.EnableNodePublicIP != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + r.Spec.EnableNodePublicIP, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.EnableNodePublicIP != nil { + if r.Spec.EnableNodePublicIP == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + r.Spec.EnableNodePublicIP, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.EnableNodePublicIP != *old.Spec.EnableNodePublicIP { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + r.Spec.EnableNodePublicIP, + "field is immutable")) + } + } + + if old.Spec.OsDiskType == nil && r.Spec.OsDiskType != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OsDiskType"), + r.Spec.OsDiskType, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.OsDiskType != nil { + if r.Spec.OsDiskType == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OsDiskType"), + r.Spec.OsDiskType, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.OsDiskType != *old.Spec.OsDiskType { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OsDiskType"), + r.Spec.OsDiskType, + "field is immutable")) + } + } + + if old.Spec.ScaleSetPriority == nil && r.Spec.ScaleSetPriority != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "ScaleSetPriority"), + r.Spec.ScaleSetPriority, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.ScaleSetPriority != nil { + if r.Spec.ScaleSetPriority == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "ScaleSetPriority"), + r.Spec.ScaleSetPriority, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.ScaleSetPriority != *old.Spec.ScaleSetPriority { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "ScaleSetPriority"), + r.Spec.ScaleSetPriority, "field is immutable")) } } + if !reflect.DeepEqual(r.Spec.AvailabilityZones, old.Spec.AvailabilityZones) { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "AvailabilityZones"), + r.Spec.AvailabilityZones, + "field is immutable")) + } + if r.Spec.Mode != string(NodePoolModeSystem) && old.Spec.Mode == string(NodePoolModeSystem) { // validate for last system node pool if err := r.validateLastSystemNodePool(client); err != nil { @@ -104,7 +317,7 @@ func (r *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePool").GroupKind(), r.Name, allErrs) } - return nil + return r.ValidateCreate(client) } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go index 81f72ae7f6e..85ee3081cf2 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -60,6 +60,94 @@ func TestAzureManagedMachinePoolDefaultingWebhook(t *testing.T) { g.Expect(*ammp.Spec.Name).To(Equal("barName")) } +func TestAzureManagedMachinePoolValidateCreateWebhook(t *testing.T) { + g := NewWithT(t) + + t.Logf("Testing ValidateCreate webhook") + + tests := []struct { + name string + pool *AzureManagedMachinePool + wantErr bool + }{ + { + name: "AutoScaling enabled, expect both MinCount and MaxCount to be set", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + AutoScaling: &AutoScaling{ + MinCount: to.Int32Ptr(2), + MaxCount: to.Int32Ptr(5), + }, + }, + }, + wantErr: false, + }, + { + name: "Node taint should follow patten key=value:NoSchedule", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + NodeTaints: []string{"key1=value1:NoSchedule"}, + }, + }, + wantErr: false, + }, + { + name: "Node taint should follow patten key=value:NoSchedule", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + NodeTaints: []string{"key1=value1NoSchedule"}, + }, + }, + wantErr: true, + }, + { + name: "AllowedUnsafeSysctls should be one of \"kernel.shm*\", \"kernel.msg*\", \"kernel.sem\", \"fs.mqueue.*\", \"net.*\"", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + KubeletConfig: &KubeletConfig{AllowedUnsafeSysctls: &[]string{"net.*"}}, + }, + }, + wantErr: false, + }, + { + name: "AllowedUnsafeSysctls should be one of \"kernel.shm*\", \"kernel.msg*\", \"kernel.sem\", \"fs.mqueue.*\", \"net.*\"", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "User", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + KubeletConfig: &KubeletConfig{AllowedUnsafeSysctls: &[]string{"net4.*"}}, + }, + }, + wantErr: true, + }, + } + + var client client.Client + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.pool.ValidateCreate(client) + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { g := NewWithT(t) @@ -107,6 +195,126 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { }, wantErr: true, }, + { + name: "Cannot change EnableFIPS of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableFIPS: to.BoolPtr(true), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableFIPS: to.BoolPtr(false), + }, + }, + wantErr: true, + }, + { + name: "Cannot change EnableNodePublicIP of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableNodePublicIP: to.BoolPtr(true), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableNodePublicIP: to.BoolPtr(false), + }, + }, + wantErr: true, + }, + { + name: "Cannot change OsDiskType of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + OsDiskType: to.StringPtr("Managed"), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + OsDiskType: to.StringPtr("Ephemeral"), + }, + }, + wantErr: true, + }, + { + name: "Cannot change ScaleSetPriority of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + ScaleSetPriority: to.StringPtr("Regular"), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + ScaleSetPriority: to.StringPtr("Spot"), + }, + }, + wantErr: true, + }, + { + name: "Cannot change MaxPods of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxPods: to.Int32Ptr(50), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxPods: to.Int32Ptr(40), + }, + }, + wantErr: true, + }, + { + name: "Cannot change NodeTaints of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + NodeTaints: []string{"key1=value1:NoSchedule"}, + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + NodeTaints: []string{"key2=value2:NoSchedule"}, + }, + }, + wantErr: true, + }, } var client client.Client for _, tc := range tests { diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 68e1ba653e1..9e0e2fce805 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -85,6 +85,31 @@ func (in *APIServerAccessProfile) DeepCopy() *APIServerAccessProfile { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoScaling) DeepCopyInto(out *AutoScaling) { + *out = *in + if in.MaxCount != nil { + in, out := &in.MaxCount, &out.MaxCount + *out = new(int32) + **out = **in + } + if in.MinCount != nil { + in, out := &in.MinCount, &out.MinCount + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoScaling. +func (in *AutoScaling) DeepCopy() *AutoScaling { + if in == nil { + return nil + } + out := new(AutoScaling) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureMachinePool) DeepCopyInto(out *AzureMachinePool) { *out = *in @@ -765,6 +790,71 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = make([]string, len(*in)) copy(*out, *in) } + if in.AutoScaling != nil { + in, out := &in.AutoScaling, &out.AutoScaling + *out = new(AutoScaling) + (*in).DeepCopyInto(*out) + } + if in.EnableNodePublicIP != nil { + in, out := &in.EnableNodePublicIP, &out.EnableNodePublicIP + *out = new(bool) + **out = **in + } + if in.EnableFIPS != nil { + in, out := &in.EnableFIPS, &out.EnableFIPS + *out = new(bool) + **out = **in + } + if in.OsDiskType != nil { + in, out := &in.OsDiskType, &out.OsDiskType + *out = new(string) + **out = **in + } + if in.NodeLabels != nil { + in, out := &in.NodeLabels, &out.NodeLabels + *out = make(map[string]*string, len(*in)) + for key, val := range *in { + var outVal *string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(string) + **out = **in + } + (*out)[key] = outVal + } + } + if in.NodeTaints != nil { + in, out := &in.NodeTaints, &out.NodeTaints + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.VnetSubnetID != nil { + in, out := &in.VnetSubnetID, &out.VnetSubnetID + *out = new(string) + **out = **in + } + if in.AvailabilityZones != nil { + in, out := &in.AvailabilityZones, &out.AvailabilityZones + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ScaleSetPriority != nil { + in, out := &in.ScaleSetPriority, &out.ScaleSetPriority + *out = new(string) + **out = **in + } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } + if in.KubeletConfig != nil { + in, out := &in.KubeletConfig, &out.KubeletConfig + *out = new(KubeletConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolSpec. @@ -802,6 +892,80 @@ func (in *AzureManagedMachinePoolStatus) DeepCopy() *AzureManagedMachinePoolStat return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfig) DeepCopyInto(out *KubeletConfig) { + *out = *in + if in.CPUManagerPolicy != nil { + in, out := &in.CPUManagerPolicy, &out.CPUManagerPolicy + *out = new(string) + **out = **in + } + if in.CPUCfsQuota != nil { + in, out := &in.CPUCfsQuota, &out.CPUCfsQuota + *out = new(bool) + **out = **in + } + if in.CPUCfsQuotaPeriod != nil { + in, out := &in.CPUCfsQuotaPeriod, &out.CPUCfsQuotaPeriod + *out = new(string) + **out = **in + } + if in.ImageGcHighThreshold != nil { + in, out := &in.ImageGcHighThreshold, &out.ImageGcHighThreshold + *out = new(int32) + **out = **in + } + if in.ImageGcLowThreshold != nil { + in, out := &in.ImageGcLowThreshold, &out.ImageGcLowThreshold + *out = new(int32) + **out = **in + } + if in.TopologyManagerPolicy != nil { + in, out := &in.TopologyManagerPolicy, &out.TopologyManagerPolicy + *out = new(string) + **out = **in + } + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = new([]string) + if **in != nil { + in, out := *in, *out + *out = make([]string, len(*in)) + copy(*out, *in) + } + } + if in.FailSwapOn != nil { + in, out := &in.FailSwapOn, &out.FailSwapOn + *out = new(bool) + **out = **in + } + if in.ContainerLogMaxSizeMB != nil { + in, out := &in.ContainerLogMaxSizeMB, &out.ContainerLogMaxSizeMB + *out = new(int32) + **out = **in + } + if in.ContainerLogMaxFiles != nil { + in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles + *out = new(int32) + **out = **in + } + if in.PodMaxPids != nil { + in, out := &in.PodMaxPids, &out.PodMaxPids + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfig. +func (in *KubeletConfig) DeepCopy() *KubeletConfig { + if in == nil { + return nil + } + out := new(KubeletConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerProfile) DeepCopyInto(out *LoadBalancerProfile) { *out = *in