From d54dd10c3e3932d9cfc82cf5258453fc57ff7dce Mon Sep 17 00:00:00 2001 From: Meixing Le Date: Mon, 20 Sep 2021 10:32:30 -0700 Subject: [PATCH] f --- api/v1alpha4/types.go | 26 +++ azure/scope/managedcontrolplane.go | 100 ++++++++ azure/services/agentpools/agentpools.go | 56 +++++ .../managedclusters/managedclusters.go | 57 ++++- azure/types.go | 39 ++++ .../v1alpha3/azuremanagedmachinepool_types.go | 81 +++++++ .../v1alpha4/azuremanagedmachinepool_types.go | 80 +++++++ .../azuremanagedmachinepool_webhook.go | 186 ++++++++++++++- .../azuremanagedmachinepool_webhook_test.go | 221 ++++++++++++++++++ 9 files changed, 844 insertions(+), 2 deletions(-) diff --git a/api/v1alpha4/types.go b/api/v1alpha4/types.go index 1ea9430e7d45..aa81e8363751 100644 --- a/api/v1alpha4/types.go +++ b/api/v1alpha4/types.go @@ -651,6 +651,32 @@ type AzureBastion struct { PublicIP PublicIPSpec `json:"publicIP,omitempty"` } +// KubeletConfig kubelet configurations of agent nodes. +type KubeletConfig struct { + // CPUManagerPolicy - CPU Manager policy to use. + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + // CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits. + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + // TopologyManagerPolicy - Topology Manager policy to use. + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` + // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"` + // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // PodMaxPids - The maximum number of processes per pod. + PodMaxPids *int32 `json:"podMaxPids,omitempty"` +} + // IsTerminalProvisioningState returns true if the ProvisioningState is a terminal state for an Azure resource. func IsTerminalProvisioningState(state ProvisioningState) bool { return state == Failed || state == Succeeded diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 5cd98970fb60..9286bb59febf 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -467,6 +467,54 @@ func (s *ManagedControlPlaneScope) GetSystemAgentPoolSpecs(ctx context.Context) ammp.Replicas = *ownerPool.Spec.Replicas } + if pool.Spec.MaxCount != nil { + ammp.MaxCount = pool.Spec.MaxCount + } + + if pool.Spec.MinCount != nil { + ammp.MinCount = pool.Spec.MinCount + } + + if pool.Spec.EnableAutoScaling != nil { + ammp.EnableAutoScaling = pool.Spec.EnableAutoScaling + } + + if pool.Spec.EnableFIPS != nil { + ammp.EnableFIPS = pool.Spec.EnableFIPS + } + + if pool.Spec.EnableNodePublicIP != nil { + ammp.EnableNodePublicIP = pool.Spec.EnableNodePublicIP + } + + if pool.Spec.NodeLabels != nil { + ammp.NodeLabels = pool.Spec.NodeLabels + } + + if pool.Spec.NodeTaints != nil { + ammp.NodeTaints = pool.Spec.NodeTaints + } + + if pool.Spec.OsDiskType != nil { + ammp.OsDiskType = pool.Spec.OsDiskType + } + + if pool.Spec.AvailabilityZones != nil { + ammp.AvailabilityZones = pool.Spec.AvailabilityZones + } + + if pool.Spec.VnetSubnetID != nil { + ammp.VnetSubnetID = *pool.Spec.VnetSubnetID + } + + if pool.Spec.MaxPods != nil { + ammp.MaxPods = pool.Spec.MaxPods + } + + if pool.Spec.KubeletConfig != nil { + ammp.KubeletConfig = (*infrav1.KubeletConfig)(pool.Spec.KubeletConfig) + } + ammps = append(ammps, ammp) } @@ -506,6 +554,58 @@ func (s *ManagedControlPlaneScope) AgentPoolSpec() azure.AgentPoolSpec { agentPoolSpec.OSDiskSizeGB = *s.InfraMachinePool.Spec.OSDiskSizeGB } + if s.InfraMachinePool.Spec.MaxCount != nil { + agentPoolSpec.MaxCount = s.InfraMachinePool.Spec.MaxCount + } + + if s.InfraMachinePool.Spec.MinCount != nil { + agentPoolSpec.MinCount = s.InfraMachinePool.Spec.MinCount + } + + if s.InfraMachinePool.Spec.EnableAutoScaling != nil { + agentPoolSpec.EnableAutoScaling = s.InfraMachinePool.Spec.EnableAutoScaling + } + + if s.InfraMachinePool.Spec.EnableFIPS != nil { + agentPoolSpec.EnableFIPS = s.InfraMachinePool.Spec.EnableFIPS + } + + if s.InfraMachinePool.Spec.EnableNodePublicIP != nil { + agentPoolSpec.EnableNodePublicIP = s.InfraMachinePool.Spec.EnableNodePublicIP + } + + if s.InfraMachinePool.Spec.NodeLabels != nil { + agentPoolSpec.NodeLabels = s.InfraMachinePool.Spec.NodeLabels + } + + if s.InfraMachinePool.Spec.NodeTaints != nil { + agentPoolSpec.NodeTaints = s.InfraMachinePool.Spec.NodeTaints + } + + if s.InfraMachinePool.Spec.OsDiskType != nil { + agentPoolSpec.OsDiskType = s.InfraMachinePool.Spec.OsDiskType + } + + if s.InfraMachinePool.Spec.VnetSubnetID != nil { + agentPoolSpec.VnetSubnetID = *s.InfraMachinePool.Spec.VnetSubnetID + } + + if s.InfraMachinePool.Spec.AvailabilityZones != nil { + agentPoolSpec.AvailabilityZones = s.InfraMachinePool.Spec.AvailabilityZones + } + + if s.InfraMachinePool.Spec.ScaleSetPriority != nil { + agentPoolSpec.ScaleSetPriority = s.InfraMachinePool.Spec.ScaleSetPriority + } + + if s.InfraMachinePool.Spec.MaxPods != nil { + agentPoolSpec.MaxPods = s.InfraMachinePool.Spec.MaxPods + } + + if s.InfraMachinePool.Spec.KubeletConfig != nil { + agentPoolSpec.KubeletConfig = (*infrav1.KubeletConfig)(s.InfraMachinePool.Spec.KubeletConfig) + } + return agentPoolSpec } diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index d5eecd13583c..845fc55fde43 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -77,6 +77,54 @@ func (s *Service) Reconcile(ctx context.Context) error { }, } + if agentPoolSpec.MaxCount != nil { + profile.MaxCount = agentPoolSpec.MaxCount + } + + if agentPoolSpec.MinCount != nil { + profile.MinCount = agentPoolSpec.MinCount + } + + if agentPoolSpec.EnableAutoScaling != nil { + profile.EnableAutoScaling = agentPoolSpec.EnableAutoScaling + } + + if agentPoolSpec.EnableFIPS != nil { + profile.EnableFIPS = agentPoolSpec.EnableFIPS + } + + if agentPoolSpec.EnableNodePublicIP != nil { + profile.EnableNodePublicIP = agentPoolSpec.EnableNodePublicIP + } + + if agentPoolSpec.NodeLabels != nil { + profile.NodeLabels = agentPoolSpec.NodeLabels + } + + if agentPoolSpec.NodeTaints != nil { + profile.NodeTaints = &agentPoolSpec.NodeTaints + } + + if agentPoolSpec.OsDiskType != nil { + profile.OsDiskType = containerservice.OSDiskType(*agentPoolSpec.OsDiskType) + } + + if agentPoolSpec.AvailabilityZones != nil { + profile.AvailabilityZones = &agentPoolSpec.AvailabilityZones + } + + if agentPoolSpec.ScaleSetPriority != nil { + profile.ScaleSetPriority = containerservice.ScaleSetPriority(*agentPoolSpec.ScaleSetPriority) + } + + if agentPoolSpec.MaxPods != nil { + profile.MaxPods = agentPoolSpec.MaxPods + } + + if agentPoolSpec.KubeletConfig != nil { + profile.KubeletConfig = (*containerservice.KubeletConfig)(agentPoolSpec.KubeletConfig) + } + existingPool, err := s.Client.Get(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name) if err != nil && !azure.ResourceNotFound(err) { return errors.Wrap(err, "failed to get existing agent pool") @@ -106,6 +154,10 @@ func (s *Service) Reconcile(ctx context.Context) error { Count: existingPool.Count, OrchestratorVersion: existingPool.OrchestratorVersion, Mode: existingPool.Mode, + MaxCount: existingPool.MaxCount, + MinCount: existingPool.MinCount, + EnableAutoScaling: existingPool.EnableAutoScaling, + NodeLabels: existingPool.NodeLabels, }, } @@ -114,6 +166,10 @@ func (s *Service) Reconcile(ctx context.Context) error { Count: profile.Count, OrchestratorVersion: profile.OrchestratorVersion, Mode: profile.Mode, + MaxCount: profile.MaxCount, + MinCount: profile.MinCount, + EnableAutoScaling: profile.EnableAutoScaling, + NodeLabels: profile.NodeLabels, }, } diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index f2fb4053ebff..8a457ec4971f 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -67,6 +67,7 @@ func New(scope ManagedClusterScope) *Service { } // Reconcile idempotently creates or updates a managed cluster, if possible. +//gocyclo:ignore func (s *Service) Reconcile(ctx context.Context) error { ctx, span := tele.Tracer().Start(ctx, "managedclusters.Service.Reconcile") defer span.End() @@ -160,9 +161,63 @@ func (s *Service) Reconcile(ctx context.Context) error { OsDiskSizeGB: &pool.OSDiskSizeGB, Count: &pool.Replicas, Type: containerservice.AgentPoolTypeVirtualMachineScaleSets, - VnetSubnetID: &managedClusterSpec.VnetSubnetID, Mode: containerservice.AgentPoolModeSystem, } + + if pool.VnetSubnetID != "" { + profile.VnetSubnetID = &pool.VnetSubnetID + } else { + profile.VnetSubnetID = &managedClusterSpec.VnetSubnetID + } + + if pool.MaxCount != nil { + profile.MaxCount = pool.MaxCount + } + + if pool.MinCount != nil { + profile.MinCount = pool.MinCount + } + + if pool.EnableAutoScaling != nil { + profile.EnableAutoScaling = pool.EnableAutoScaling + } + + if pool.EnableFIPS != nil { + profile.EnableFIPS = pool.EnableFIPS + } + + if pool.EnableNodePublicIP != nil { + profile.EnableNodePublicIP = pool.EnableNodePublicIP + } + + if pool.NodeLabels != nil { + profile.NodeLabels = pool.NodeLabels + } + + if pool.NodeTaints != nil { + profile.NodeTaints = &pool.NodeTaints + } + + if pool.OsDiskType != nil { + profile.OsDiskType = containerservice.OSDiskType(*pool.OsDiskType) + } + + if pool.AvailabilityZones != nil { + profile.AvailabilityZones = &pool.AvailabilityZones + } + + if pool.ScaleSetPriority != nil { + profile.ScaleSetPriority = containerservice.ScaleSetPriority(*pool.ScaleSetPriority) + } + + if pool.MaxPods != nil { + profile.MaxPods = pool.MaxPods + } + + if pool.KubeletConfig != nil { + profile.KubeletConfig = (*containerservice.KubeletConfig)(pool.KubeletConfig) + } + *managedCluster.AgentPoolProfiles = append(*managedCluster.AgentPoolProfiles, profile) } diff --git a/azure/types.go b/azure/types.go index c10d46b45b07..59baa39bdda1 100644 --- a/azure/types.go +++ b/azure/types.go @@ -409,4 +409,43 @@ type AgentPoolSpec struct { // Mode represents mode of an agent pool. Possible values include: 'System', 'User'. Mode string + + // Max count for auto scaling + MaxCount *int32 `json:"maxCount,omitempty"` + + // Min count for auto scaling + MinCount *int32 `json:"minCount,omitempty"` + + // Enable auto scaling + EnableAutoScaling *bool `json:"EnableAutoScaling,omitempty"` + + // Enable FIPS node image + EnableFIPS *bool `json:"EnableFIPS,omitempty"` + + // Enable node public IP + EnableNodePublicIP *bool `json:"EnableNodePublicIP,omitempty"` + + // Node labels + NodeLabels map[string]*string `json:"NodeLabels,omitempty"` + + // Node taints + NodeTaints []string `json:"NodeTaints,omitempty"` + + // Node OS disk type + OsDiskType *string `json:"OsDiskType,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular' + // +optional + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // MaxPods - Maximum number of pods that can run on a node. + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + + // KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes. + // +optional + KubeletConfig *infrav1.KubeletConfig `json:"kubeletConfig,omitempty"` } diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_types.go b/exp/api/v1alpha3/azuremanagedmachinepool_types.go index cfdd986a31fc..1e22a160ff90 100644 --- a/exp/api/v1alpha3/azuremanagedmachinepool_types.go +++ b/exp/api/v1alpha3/azuremanagedmachinepool_types.go @@ -37,6 +37,87 @@ type AzureManagedMachinePoolSpec struct { // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` + + // MaxCount - Maximum number of nodes for auto-scaling + // +optional + MaxCount *int32 `json:"maxCount,omitempty"` + + // MinCount - Minimum number of nodes for auto-scaling + // +optional + MinCount *int32 `json:"minCount,omitempty"` + + // EnableAutoScaling - Whether to enable auto-scaler + // +optional + EnableAutoScaling *bool `json:"enableAutoScaling,omitempty"` + + // EnableNodePublicIP - Enable public IP for nodes + // +optional + EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + + // EnableFIPS - Whether to use FIPS enabled OS + // +optional + EnableFIPS *bool `json:"enableFIPS,omitempty"` + + // OsDiskType - OS disk type to be used for machines in a given agent pool. Allowed values are 'Ephemeral' and 'Managed'. If unspecified, defaults to 'Ephemeral' when the VM supports ephemeral OS and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to 'Managed'. May not be changed after creation. Possible values include: 'Managed', 'Ephemeral' + // +kubebuilder:validation:Enum=Managed;Ephemeral + // +optional + OsDiskType *string `json:"osDiskType,omitempty"` + + // NodeLabels - Agent pool node labels to be persisted across all nodes in agent pool. + // +optional + NodeLabels map[string]*string `json:"nodeLabels,omitempty"` + + // NodeTaints - Taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. + // +optional + NodeTaints []string `json:"nodeTaints,omitempty"` + + // VnetSubnetID - VNet SubnetID specifies the VNet's subnet identifier for nodes and maybe pods + // +optional + VnetSubnetID *string `json:"vnetSubnetID,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + // +optional + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular' + // +kubebuilder:validation:Enum=Regular;Spot + // +optional + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // MaxPods - Maximum number of pods that can run on a node. + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + + // KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes. + // +optional + KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` +} + +// KubeletConfig kubelet configurations of agent nodes. +type KubeletConfig struct { + // CPUManagerPolicy - CPU Manager policy to use. + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + // CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits. + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + // TopologyManagerPolicy - Topology Manager policy to use. + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` + // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + // TODO: consider using []string instead of *[]string + AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"` + // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // PodMaxPids - The maximum number of processes per pod. + PodMaxPids *int32 `json:"podMaxPids,omitempty"` } // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool. diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_types.go b/exp/api/v1alpha4/azuremanagedmachinepool_types.go index 68f704307f7f..26520c3db5df 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_types.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_types.go @@ -52,6 +52,86 @@ type AzureManagedMachinePoolSpec struct { // ProviderIDList is the unique identifier as specified by the cloud provider. // +optional ProviderIDList []string `json:"providerIDList,omitempty"` + + // MaxCount - Maximum number of nodes for auto-scaling + // +optional + MaxCount *int32 `json:"maxCount,omitempty"` + + // MinCount - Minimum number of nodes for auto-scaling + // +optional + MinCount *int32 `json:"minCount,omitempty"` + + // EnableAutoScaling - Whether to enable auto-scaler + // +optional + EnableAutoScaling *bool `json:"enableAutoScaling,omitempty"` + + // EnableNodePublicIP - Enable public IP for nodes + // +optional + EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` + + // EnableFIPS - Whether to use FIPS enabled OS + // +optional + EnableFIPS *bool `json:"enableFIPS,omitempty"` + + // OsDiskType - OS disk type to be used for machines in a given agent pool. Allowed values are 'Ephemeral' and 'Managed'. If unspecified, defaults to 'Ephemeral' when the VM supports ephemeral OS and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to 'Managed'. May not be changed after creation. Possible values include: 'Managed', 'Ephemeral' + // +kubebuilder:validation:Enum=Managed;Ephemeral + // +optional + OsDiskType *string `json:"osDiskType,omitempty"` + + // NodeLabels - Agent pool node labels to be persisted across all nodes in agent pool. + // +optional + NodeLabels map[string]*string `json:"nodeLabels,omitempty"` + + // NodeTaints - Taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. + // +optional + NodeTaints []string `json:"nodeTaints,omitempty"` + + // VnetSubnetID - VNet SubnetID specifies the VNet's subnet identifier for nodes and maybe pods + // +optional + VnetSubnetID *string `json:"vnetSubnetID,omitempty"` + + // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + // +optional + AvailabilityZones []string `json:"availabilityZones,omitempty"` + + // ScaleSetPriority - ScaleSetPriority to be used to specify virtual machine scale set priority. Default to regular. Possible values include: 'Spot', 'Regular' + // +kubebuilder:validation:Enum=Regular;Spot + // +optional + ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` + + // MaxPods - Maximum number of pods that can run on a node. + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + + // KubeletConfig - KubeletConfig specifies the configuration of kubelet on agent nodes. + // +optional + KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` +} + +// KubeletConfig kubelet configurations of agent nodes. +type KubeletConfig struct { + // CPUManagerPolicy - CPU Manager policy to use. + CPUManagerPolicy *string `json:"cpuManagerPolicy,omitempty"` + // CPUCfsQuota - Enable CPU CFS quota enforcement for containers that specify CPU limits. + CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` + // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` + // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` + // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + ImageGcLowThreshold *int32 `json:"imageGcLowThreshold,omitempty"` + // TopologyManagerPolicy - Topology Manager policy to use. + TopologyManagerPolicy *string `json:"topologyManagerPolicy,omitempty"` + // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + AllowedUnsafeSysctls *[]string `json:"allowedUnsafeSysctls,omitempty"` + // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` + // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // PodMaxPids - The maximum number of processes per pod. + PodMaxPids *int32 `json:"podMaxPids,omitempty"` } // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool. diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_webhook.go b/exp/api/v1alpha4/azuremanagedmachinepool_webhook.go index d6514ce1fbdf..eebdc5701656 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_webhook.go @@ -18,6 +18,7 @@ package v1alpha4 import ( "context" + "reflect" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -51,10 +52,33 @@ func (r *AzureManagedMachinePool) Default(client client.Client) { // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. func (r *AzureManagedMachinePool) ValidateCreate(client client.Client) error { azuremanagedmachinepoollog.Info("validate create", "name", r.Name) + var allErrs field.ErrorList + // If AutoScaling is disabled, both MinCount and MaxCount should not be set + if (r.Spec.MaxCount != nil || r.Spec.MinCount != nil) && (r.Spec.EnableAutoScaling == nil || !*r.Spec.EnableAutoScaling) { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableAutoScaling"), + r.Spec.EnableAutoScaling, + "MaxCount and MinCount cannot be set when AutoScaling is disabled")) + } + + // If AutoScaling is enabled, both MinCount and MaxCount should be set + if r.Spec.EnableAutoScaling != nil && (r.Spec.MaxCount == nil || r.Spec.MinCount == nil) && *r.Spec.EnableAutoScaling { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableAutoScaling"), + r.Spec.EnableAutoScaling, + "MaxCount and MinCount must be set when AutoScaling is enabled")) + } + + if len(allErrs) != 0 { + return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePool").GroupKind(), r.Name, allErrs) + } return nil } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +//gocyclo:ignore func (r *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client client.Client) error { old := oldRaw.(*AzureManagedMachinePool) var allErrs field.ErrorList @@ -86,6 +110,166 @@ func (r *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c } } + if !reflect.DeepEqual(r.Spec.NodeTaints, old.Spec.NodeTaints) { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "NodeTaints"), + r.Spec.NodeTaints, + "field is immutable")) + } + + if old.Spec.VnetSubnetID == nil && r.Spec.VnetSubnetID != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "VnetSubnetID"), + r.Spec.VnetSubnetID, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.VnetSubnetID != nil { + if r.Spec.VnetSubnetID == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "VnetSubnetID"), + r.Spec.VnetSubnetID, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.VnetSubnetID != *old.Spec.VnetSubnetID { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "VnetSubnetID"), + r.Spec.VnetSubnetID, + "field is immutable")) + } + } + + if old.Spec.MaxPods == nil && r.Spec.MaxPods != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MaxPods"), + r.Spec.MaxPods, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.MaxPods != nil { + if r.Spec.MaxPods == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MaxPods"), + r.Spec.MaxPods, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.MaxPods != *old.Spec.MaxPods { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "MaxPods"), + r.Spec.MaxPods, + "field is immutable")) + } + } + + if old.Spec.EnableFIPS == nil && r.Spec.EnableFIPS != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableFIPS"), + r.Spec.EnableFIPS, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.EnableFIPS != nil { + if r.Spec.EnableFIPS == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableFIPS"), + r.Spec.EnableFIPS, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.EnableFIPS != *old.Spec.EnableFIPS { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableFIPS"), + r.Spec.EnableFIPS, + "field is immutable")) + } + } + + if old.Spec.EnableNodePublicIP == nil && r.Spec.EnableNodePublicIP != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + r.Spec.EnableNodePublicIP, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.EnableNodePublicIP != nil { + if r.Spec.EnableNodePublicIP == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + r.Spec.EnableNodePublicIP, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.EnableNodePublicIP != *old.Spec.EnableNodePublicIP { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "EnableNodePublicIP"), + r.Spec.EnableNodePublicIP, + "field is immutable")) + } + } + + if old.Spec.OsDiskType == nil && r.Spec.OsDiskType != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OsDiskType"), + r.Spec.OsDiskType, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.OsDiskType != nil { + if r.Spec.OsDiskType == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OsDiskType"), + r.Spec.OsDiskType, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.OsDiskType != *old.Spec.OsDiskType { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "OsDiskType"), + r.Spec.OsDiskType, + "field is immutable")) + } + } + + if old.Spec.ScaleSetPriority == nil && r.Spec.ScaleSetPriority != nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "ScaleSetPriority"), + r.Spec.ScaleSetPriority, + "field is immutable, setting after creation is not allowed")) + } + + if old.Spec.ScaleSetPriority != nil { + if r.Spec.ScaleSetPriority == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "ScaleSetPriority"), + r.Spec.ScaleSetPriority, + "field is immutable, unsetting is not allowed")) + } else if *r.Spec.ScaleSetPriority != *old.Spec.ScaleSetPriority { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "ScaleSetPriority"), + r.Spec.ScaleSetPriority, + "field is immutable")) + } + } + + if !reflect.DeepEqual(r.Spec.AvailabilityZones, old.Spec.AvailabilityZones) { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "AvailabilityZones"), + r.Spec.AvailabilityZones, + "field is immutable")) + } + if r.Spec.Mode != string(NodePoolModeSystem) && old.Spec.Mode == string(NodePoolModeSystem) { // validate for last system node pool if err := r.validateLastSystemNodePool(client); err != nil { @@ -100,7 +284,7 @@ func (r *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePool").GroupKind(), r.Name, allErrs) } - return nil + return r.ValidateCreate(client) } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_webhook_test.go b/exp/api/v1alpha4/azuremanagedmachinepool_webhook_test.go index a05506f8106c..cfd0f95ee764 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_webhook_test.go @@ -47,6 +47,107 @@ func TestAzureManagedMachinePoolDefaultingWebhook(t *testing.T) { g.Expect(val).To(Equal("System")) } +func TestAzureManagedMachinePoolValidateCreateWebhook(t *testing.T) { + g := NewWithT(t) + + t.Logf("Testing ValidateCreate webhook") + + tests := []struct { + name string + pool *AzureManagedMachinePool + wantErr bool + }{ + { + name: "AutoScaling enabled, expect both MinCount and MaxCount to be set", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MinCount: to.Int32Ptr(2), + MaxCount: to.Int32Ptr(5), + EnableAutoScaling: to.BoolPtr(true), + }, + }, + wantErr: false, + }, + { + name: "AutoScaling disabled, expect neither MinCount nor MaxCount to be set", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableAutoScaling: to.BoolPtr(false), + }, + }, + wantErr: false, + }, + { + name: "EnableAutoScaling is nil, but maxCount is set", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxCount: to.Int32Ptr(5), + }, + }, + wantErr: true, + }, + { + name: "EnableAutoScaling is nil, but minCount is set", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MinCount: to.Int32Ptr(2), + }, + }, + wantErr: true, + }, + { + name: "EnableAutoScaling is disabled, but maxCount is set", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxCount: to.Int32Ptr(5), + EnableAutoScaling: to.BoolPtr(false), + }, + }, + wantErr: true, + }, + { + name: "EnableAutoScaling is enabled, but minCount is missing", + pool: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxCount: to.Int32Ptr(5), + EnableAutoScaling: to.BoolPtr(false), + }, + }, + wantErr: true, + }, + } + + var client client.Client + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.pool.ValidateCreate(client) + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} + func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { g := NewWithT(t) @@ -94,6 +195,126 @@ func TestAzureManagedMachinePoolUpdatingWebhook(t *testing.T) { }, wantErr: true, }, + { + name: "Cannot change EnableFIPS of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableFIPS: to.BoolPtr(true), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableFIPS: to.BoolPtr(false), + }, + }, + wantErr: true, + }, + { + name: "Cannot change EnableNodePublicIP of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableNodePublicIP: to.BoolPtr(true), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + EnableNodePublicIP: to.BoolPtr(false), + }, + }, + wantErr: true, + }, + { + name: "Cannot change OsDiskType of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + OsDiskType: to.StringPtr("Managed"), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + OsDiskType: to.StringPtr("Ephemeral"), + }, + }, + wantErr: true, + }, + { + name: "Cannot change ScaleSetPriority of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + ScaleSetPriority: to.StringPtr("Regular"), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + ScaleSetPriority: to.StringPtr("Spot"), + }, + }, + wantErr: true, + }, + { + name: "Cannot change MaxPods of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxPods: to.Int32Ptr(50), + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + MaxPods: to.Int32Ptr(40), + }, + }, + wantErr: true, + }, + { + name: "Cannot change NodeTaints of the agentpool", + new: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + NodeTaints: []string{"key1=value1:NoSchedule"}, + }, + }, + old: &AzureManagedMachinePool{ + Spec: AzureManagedMachinePoolSpec{ + Mode: "System", + SKU: "StandardD2S_V3", + OSDiskSizeGB: to.Int32Ptr(512), + NodeTaints: []string{"key2=value2:NoSchedule"}, + }, + }, + wantErr: true, + }, } var client client.Client for _, tc := range tests {